Files

47 lines
1.2 KiB
Python
Executable File

"""内容解析:HTML → 纯文本(stdlib),文本分块。"""
from __future__ import annotations
import re
from html.parser import HTMLParser
from typing import List
class _TextExtractor(HTMLParser):
SKIP = {"script", "style", "noscript"}
def __init__(self):
super().__init__()
self.parts: List[str] = []
self._skip = 0
def handle_starttag(self, tag, attrs):
if tag in self.SKIP:
self._skip += 1
def handle_endtag(self, tag):
if tag in self.SKIP and self._skip > 0:
self._skip -= 1
def handle_data(self, data):
if self._skip == 0 and data.strip():
self.parts.append(data.strip())
def html_to_text(html: str) -> str:
p = _TextExtractor()
try:
p.feed(html)
except Exception:
return re.sub(r"<[^>]+>", " ", html)
return "\n".join(p.parts)
def chunk_text(text: str, size: int = 500, overlap: int = 50) -> List[str]:
text = text.strip()
if len(text) <= size:
return [text] if text else []
chunks, start = [], 0
while start < len(text):
chunks.append(text[start:start + size])
start += size - overlap
return chunks