104 lines
4.6 KiB
Python
Executable File
104 lines
4.6 KiB
Python
Executable File
"""自主知识获取(自己找吃的):发现信息源 → 抓取清洗 → 理解结构化 → 写入知识库。
|
|
|
|
三类来源:订阅源(RSS/GitHub Release 定时巡检)、主动搜索(不懂就搜)、
|
|
对话触发(即时搜索沉淀)。全链路已可运行。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import List, Optional
|
|
|
|
from core.event_bus import EventBus
|
|
from memory.knowledge_digest import KnowledgeDigest
|
|
from memory.knowledge_garden import KnowledgeGarden
|
|
from memory.web_scraper import WebScraper
|
|
from tools.rss_reader import RSSReader
|
|
|
|
log = logging.getLogger("taichu.forager")
|
|
|
|
MAX_ITEMS_PER_SOURCE = 5
|
|
|
|
|
|
class KnowledgeForager:
|
|
def __init__(self, digest: KnowledgeDigest, garden: KnowledgeGarden,
|
|
scraper: Optional[WebScraper] = None,
|
|
bus: Optional[EventBus] = None,
|
|
sources_cfg: Optional[dict] = None,
|
|
search_tool=None,
|
|
rss_reader: Optional[RSSReader] = None):
|
|
self.digest = digest
|
|
self.garden = garden
|
|
self.scraper = scraper or WebScraper()
|
|
self.bus = bus
|
|
self.sources_cfg = sources_cfg or {}
|
|
self.search_tool = search_tool
|
|
self.rss_reader = rss_reader or RSSReader()
|
|
|
|
def discover(self) -> List[dict]:
|
|
"""发现待抓取信息源(启用中的订阅源清单)。"""
|
|
out = []
|
|
for feed in self.sources_cfg.get("rss_feeds", []):
|
|
if feed.get("enabled"):
|
|
out.append({"kind": "rss", **feed})
|
|
for repo in self.sources_cfg.get("github_releases", []):
|
|
if repo.get("enabled"):
|
|
out.append({"kind": "github_release", **repo})
|
|
return out
|
|
|
|
def _ingest_item(self, item: dict, scrape_link: bool) -> bool:
|
|
title, text, link = item.get("title", ""), item.get("summary", ""), item.get("link", "")
|
|
if scrape_link and link:
|
|
page = self.scraper.fetch(link)
|
|
if page and page["text"]:
|
|
text = page["text"]
|
|
title = page["title"] or title
|
|
if not text:
|
|
return False
|
|
res = self.ingest_content(title, text, link)
|
|
return res.get("status") == "accepted"
|
|
|
|
def forage(self, topic: Optional[str] = None) -> dict:
|
|
"""一轮觅食:订阅巡检 + 主题主动搜索。"""
|
|
result = {"sources": 0, "ingested": 0, "search": None}
|
|
sources = self.discover()
|
|
result["sources"] = len(sources)
|
|
for src in sources:
|
|
try:
|
|
if src["kind"] == "rss":
|
|
items = self.rss_reader.fetch(src["url"]) or []
|
|
for item in items[:MAX_ITEMS_PER_SOURCE]:
|
|
if self._ingest_item(item, scrape_link=True):
|
|
result["ingested"] += 1
|
|
elif src["kind"] == "github_release":
|
|
items = self.rss_reader.fetch_github_release(
|
|
src["repo"], api_base=src.get("api_base", "https://api.github.com")) or []
|
|
for item in items[:MAX_ITEMS_PER_SOURCE]:
|
|
# Release 正文本身即内容,无需再抓页面
|
|
if self._ingest_item(item, scrape_link=False):
|
|
result["ingested"] += 1
|
|
except Exception:
|
|
log.exception("订阅源处理异常 %s", src.get("name") or src.get("repo"))
|
|
if topic and self.search_tool is not None:
|
|
search_res = self.search_tool.search(topic)
|
|
result["search"] = search_res
|
|
for hit in search_res.get("results", [])[:3]:
|
|
page = self.scraper.fetch(hit.get("url", ""))
|
|
if page and page["text"]:
|
|
r = self.ingest_content(page["title"], page["text"], page["url"])
|
|
if r.get("status") == "accepted":
|
|
result["ingested"] += 1
|
|
if self.bus:
|
|
self.bus.emit("knowledge.forage_done", result, source="forager")
|
|
return result
|
|
|
|
def ingest_content(self, title: str, text: str, source_url: str = "") -> dict:
|
|
"""对话触发/注入内容的即时消化入库。"""
|
|
d = self.digest.digest(title, text, source_url)
|
|
item = {"title": title, "content": d.get("summary") or text[:500],
|
|
"source_url": source_url, "confidence": d.get("confidence", 0.5),
|
|
"tags": d.get("tags", []), "questions": d.get("questions", [])}
|
|
res = self.garden.ingest(item)
|
|
if res.get("status") == "accepted" and self.bus:
|
|
self.bus.emit("knowledge.learned", {"title": title, "id": res.get("id")},
|
|
source="forager")
|
|
return res |