47 lines
1.2 KiB
Python
Executable File
47 lines
1.2 KiB
Python
Executable File
"""内容解析:HTML → 纯文本(stdlib),文本分块。"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from html.parser import HTMLParser
|
|
from typing import List
|
|
|
|
|
|
class _TextExtractor(HTMLParser):
|
|
SKIP = {"script", "style", "noscript"}
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.parts: List[str] = []
|
|
self._skip = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag in self.SKIP:
|
|
self._skip += 1
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag in self.SKIP and self._skip > 0:
|
|
self._skip -= 1
|
|
|
|
def handle_data(self, data):
|
|
if self._skip == 0 and data.strip():
|
|
self.parts.append(data.strip())
|
|
|
|
|
|
def html_to_text(html: str) -> str:
|
|
p = _TextExtractor()
|
|
try:
|
|
p.feed(html)
|
|
except Exception:
|
|
return re.sub(r"<[^>]+>", " ", html)
|
|
return "\n".join(p.parts)
|
|
|
|
|
|
def chunk_text(text: str, size: int = 500, overlap: int = 50) -> List[str]:
|
|
text = text.strip()
|
|
if len(text) <= size:
|
|
return [text] if text else []
|
|
chunks, start = [], 0
|
|
while start < len(text):
|
|
chunks.append(text[start:start + size])
|
|
start += size - overlap
|
|
return chunks |