Files
TaiChuSeed/memory/knowledge_forager.py

104 lines
4.6 KiB
Python
Executable File

"""自主知识获取(自己找吃的):发现信息源 → 抓取清洗 → 理解结构化 → 写入知识库。
三类来源:订阅源(RSS/GitHub Release 定时巡检)、主动搜索(不懂就搜)、
对话触发(即时搜索沉淀)。全链路已可运行。
"""
from __future__ import annotations
import logging
from typing import List, Optional
from core.event_bus import EventBus
from memory.knowledge_digest import KnowledgeDigest
from memory.knowledge_garden import KnowledgeGarden
from memory.web_scraper import WebScraper
from tools.rss_reader import RSSReader
log = logging.getLogger("taichu.forager")
MAX_ITEMS_PER_SOURCE = 5
class KnowledgeForager:
def __init__(self, digest: KnowledgeDigest, garden: KnowledgeGarden,
scraper: Optional[WebScraper] = None,
bus: Optional[EventBus] = None,
sources_cfg: Optional[dict] = None,
search_tool=None,
rss_reader: Optional[RSSReader] = None):
self.digest = digest
self.garden = garden
self.scraper = scraper or WebScraper()
self.bus = bus
self.sources_cfg = sources_cfg or {}
self.search_tool = search_tool
self.rss_reader = rss_reader or RSSReader()
def discover(self) -> List[dict]:
"""发现待抓取信息源(启用中的订阅源清单)。"""
out = []
for feed in self.sources_cfg.get("rss_feeds", []):
if feed.get("enabled"):
out.append({"kind": "rss", **feed})
for repo in self.sources_cfg.get("github_releases", []):
if repo.get("enabled"):
out.append({"kind": "github_release", **repo})
return out
def _ingest_item(self, item: dict, scrape_link: bool) -> bool:
title, text, link = item.get("title", ""), item.get("summary", ""), item.get("link", "")
if scrape_link and link:
page = self.scraper.fetch(link)
if page and page["text"]:
text = page["text"]
title = page["title"] or title
if not text:
return False
res = self.ingest_content(title, text, link)
return res.get("status") == "accepted"
def forage(self, topic: Optional[str] = None) -> dict:
"""一轮觅食:订阅巡检 + 主题主动搜索。"""
result = {"sources": 0, "ingested": 0, "search": None}
sources = self.discover()
result["sources"] = len(sources)
for src in sources:
try:
if src["kind"] == "rss":
items = self.rss_reader.fetch(src["url"]) or []
for item in items[:MAX_ITEMS_PER_SOURCE]:
if self._ingest_item(item, scrape_link=True):
result["ingested"] += 1
elif src["kind"] == "github_release":
items = self.rss_reader.fetch_github_release(
src["repo"], api_base=src.get("api_base", "https://api.github.com")) or []
for item in items[:MAX_ITEMS_PER_SOURCE]:
# Release 正文本身即内容,无需再抓页面
if self._ingest_item(item, scrape_link=False):
result["ingested"] += 1
except Exception:
log.exception("订阅源处理异常 %s", src.get("name") or src.get("repo"))
if topic and self.search_tool is not None:
search_res = self.search_tool.search(topic)
result["search"] = search_res
for hit in search_res.get("results", [])[:3]:
page = self.scraper.fetch(hit.get("url", ""))
if page and page["text"]:
r = self.ingest_content(page["title"], page["text"], page["url"])
if r.get("status") == "accepted":
result["ingested"] += 1
if self.bus:
self.bus.emit("knowledge.forage_done", result, source="forager")
return result
def ingest_content(self, title: str, text: str, source_url: str = "") -> dict:
"""对话触发/注入内容的即时消化入库。"""
d = self.digest.digest(title, text, source_url)
item = {"title": title, "content": d.get("summary") or text[:500],
"source_url": source_url, "confidence": d.get("confidence", 0.5),
"tags": d.get("tags", []), "questions": d.get("questions", [])}
res = self.garden.ingest(item)
if res.get("status") == "accepted" and self.bus:
self.bus.emit("knowledge.learned", {"title": title, "id": res.get("id")},
source="forager")
return res