918 lines
31 KiB
Python
Executable File
918 lines
31 KiB
Python
Executable File
#!/usr/bin/python3
|
|
"""AI Agent 升级迭代日报 — 抓取 + 存档 + 报告管道 (B1 阶段)."""
|
|
# LLM-INTEGRATION-POINT: B2 将替换 summarize_with_llm 函数体
|
|
# (litellm analysis, POST http://127.0.0.1:4000/v1/chat/completions,
|
|
# model="analysis", 无 auth, timeout=LLM_TIMEOUT)
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from concurrent import futures
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
# blog-app root on sys.path so the analysis service (Stage 1 refactor) is
|
|
# importable from cron/systemd contexts where CWD is not the app root.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
import config # noqa: E402 (needs the sys.path insert above)
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
stream=sys.stdout,
|
|
)
|
|
log = logging.getLogger("daily-watchdog")
|
|
|
|
BASE_DIR = Path.home() / "opencode-blog-showcase"
|
|
DATA_DIR = BASE_DIR / "data"
|
|
LOGS_DIR = BASE_DIR / "logs"
|
|
DB_PATH = DATA_DIR / "watchdog.db"
|
|
|
|
UA = "daily-watchdog/1.0"
|
|
HTTP_TIMEOUT = 30
|
|
GITHUB_TIMEOUT = 30
|
|
EXA_TIMEOUT = 60
|
|
ATOM_TIMEOUT = 45
|
|
LLM_URL = "http://127.0.0.1:4000/v1/chat/completions"
|
|
LLM_MODEL = "analysis"
|
|
LLM_TIMEOUT = 300
|
|
|
|
# Prompt trimming budgets shared with services/analysis.py (OPT-10).
|
|
LLM_BODY_LIMIT = config.LLM_BODY_LIMIT
|
|
LLM_FIELD_LIMIT = config.LLM_FIELD_LIMIT
|
|
LLM_MESSAGE_LIMIT = config.LLM_MESSAGE_LIMIT
|
|
LLM_TOTAL_LIMIT = config.LLM_TOTAL_LIMIT
|
|
|
|
# Parallel collection (OPT-9): 6 projects x 5 sources + 1 LLM call each are
|
|
# I/O bound, so threads beat the GIL. Capped at 3 to stay under GitHub's
|
|
# unauthenticated rate limit (60 req/hr) and to be polite to npm/atom.
|
|
COLLECT_MAX_WORKERS = int(os.environ.get("BLOG_APP_COLLECT_WORKERS", "3"))
|
|
# Global cap so one hanging project cannot stall the whole run:
|
|
# slowest data source (Exa, 60s + one retry) + LLM timeout + slack.
|
|
COLLECT_PROJECT_TIMEOUT = LLM_TIMEOUT + 120
|
|
|
|
CANONICAL_SOURCES = [
|
|
"GitHub Releases API",
|
|
"GitHub commits API",
|
|
"npm registry",
|
|
"GitHub releases.atom",
|
|
"Exa web search",
|
|
]
|
|
|
|
PROJECTS = [
|
|
{
|
|
"key": "opencode",
|
|
"label": "sst/opencode",
|
|
"github": "sst/opencode",
|
|
"npm": "opencode-ai",
|
|
"installed": "1.18.14",
|
|
"star": True,
|
|
},
|
|
{
|
|
"key": "claude-code",
|
|
"label": "anthropics/claude-code",
|
|
"github": "anthropics/claude-code",
|
|
"npm": "@anthropic-ai/claude-code",
|
|
"installed": None,
|
|
"star": True,
|
|
},
|
|
{
|
|
"key": "codex",
|
|
"label": "openai/codex",
|
|
"github": "openai/codex",
|
|
"npm": "@openai/codex",
|
|
"installed": None,
|
|
"star": False,
|
|
},
|
|
{
|
|
"key": "hermes-agent",
|
|
"label": "NousResearch/hermes-agent",
|
|
"github": "NousResearch/hermes-agent",
|
|
"npm": "hermes-agent",
|
|
"installed": None,
|
|
"star": True,
|
|
},
|
|
{
|
|
"key": "oh-my-openagent",
|
|
"label": "code-yeongyu/oh-my-openagent",
|
|
"github": "code-yeongyu/oh-my-openagent",
|
|
"npm": "oh-my-openagent",
|
|
"installed": None,
|
|
"star": True,
|
|
},
|
|
{
|
|
"key": "hindsight",
|
|
"label": "vectorize-io/hindsight",
|
|
"github": "vectorize-io/hindsight",
|
|
"npm": "hindsight",
|
|
"installed": None,
|
|
"star": True,
|
|
"npm_relevance_check": True,
|
|
},
|
|
]
|
|
|
|
ATOM_NS = "{http://www.w3.org/2005/Atom}"
|
|
|
|
|
|
def _llm_failure(reason: str) -> dict:
|
|
return {
|
|
"summary_zh": f"(LLM 分析失败: {reason})",
|
|
"breaking": None,
|
|
"breaking_reason": "LLM 分析失败",
|
|
"relevance_note": "",
|
|
}
|
|
|
|
|
|
def _truncate_field(value, limit: int):
|
|
"""Recursively truncate every string in a nested structure."""
|
|
if isinstance(value, str):
|
|
return value[:limit]
|
|
if isinstance(value, dict):
|
|
return {k: _truncate_field(v, limit) for k, v in value.items()}
|
|
if isinstance(value, list):
|
|
return [_truncate_field(v, limit) for v in value]
|
|
return value
|
|
|
|
|
|
def _extract_json_object(text: str) -> dict | None:
|
|
"""Extract the first {...} JSON object from text (tolerates fences/prose)."""
|
|
start = text.find("{")
|
|
end = text.rfind("}")
|
|
if start == -1 or end == -1 or end <= start:
|
|
return None
|
|
try:
|
|
obj = json.loads(text[start:end + 1])
|
|
except json.JSONDecodeError:
|
|
return None
|
|
return obj if isinstance(obj, dict) else None
|
|
|
|
|
|
def _short_reason(e: Exception, default: str = "请求失败") -> str:
|
|
"""Short single-line failure reason (≤80 chars, no newlines)."""
|
|
msg = str(e).strip().replace("\n", " ").replace("\r", " ")
|
|
if not msg:
|
|
msg = default
|
|
return msg[:80]
|
|
|
|
|
|
def summarize_with_llm(project_key: str, collected_data: dict) -> dict:
|
|
# 数据截断: release body ≤LLM_BODY_LIMIT, 其余字段各 ≤LLM_FIELD_LIMIT,
|
|
# 总数据部分 ≤LLM_TOTAL_LIMIT (12000 chars ≈ 3000 tokens, 见 config.py)
|
|
trimmed: dict = {}
|
|
for key, value in collected_data.items():
|
|
if value is None:
|
|
continue
|
|
if key == "github_release" and isinstance(value, dict):
|
|
rel = dict(value)
|
|
rel["body"] = (rel.get("body") or "")[:LLM_BODY_LIMIT]
|
|
trimmed[key] = _truncate_field(rel, LLM_FIELD_LIMIT)
|
|
trimmed[key]["body"] = (value.get("body") or "")[:LLM_BODY_LIMIT]
|
|
else:
|
|
trimmed[key] = _truncate_field(value, LLM_FIELD_LIMIT)
|
|
|
|
data_json = json.dumps(trimmed, ensure_ascii=False, default=str)
|
|
if len(data_json) > LLM_TOTAL_LIMIT:
|
|
# 极端情况下继续收缩: 砍掉 exa_results 再砍 commits
|
|
trimmed.pop("exa_results", None)
|
|
data_json = json.dumps(trimmed, ensure_ascii=False, default=str)
|
|
if len(data_json) > LLM_TOTAL_LIMIT and isinstance(trimmed.get("github_commits"), list):
|
|
trimmed["github_commits"] = trimmed["github_commits"][:3]
|
|
data_json = json.dumps(trimmed, ensure_ascii=False, default=str)
|
|
|
|
prompt = (
|
|
"你是 AI 编程助手项目的升级监控分析师。下面是项目 "
|
|
f"{project_key} 的最新抓取数据 (JSON):\n\n{data_json}\n\n"
|
|
"请分析这些变更, 严格只返回一个 JSON 对象, 不要输出其他内容:\n"
|
|
"{\n"
|
|
' "summary_zh": "不少于50字的中文摘要, 概括本次更新的核心内容, 2-3句话",\n'
|
|
' "breaking": true 或 false, 表示是否存在不向后兼容的破坏性变更,\n'
|
|
' "breaking_reason": "若有 breaking 说明具体内容; 若无则写 \'未发现破坏性变更\'",\n'
|
|
' "relevance_note": "对本地已安装该工具用户的实际意义, 1-2句话"\n'
|
|
"}"
|
|
)
|
|
messages = [
|
|
{"role": "system", "content": "你是严谨的 AI 工具升级监控分析师, 只输出 JSON。"},
|
|
{"role": "user", "content": prompt},
|
|
]
|
|
|
|
try:
|
|
resp = requests.post(
|
|
LLM_URL,
|
|
json={"model": LLM_MODEL, "messages": messages},
|
|
timeout=LLM_TIMEOUT,
|
|
)
|
|
resp.raise_for_status()
|
|
content = (
|
|
resp.json()
|
|
.get("choices", [{}])[0]
|
|
.get("message", {})
|
|
.get("content", "")
|
|
)
|
|
if not content:
|
|
log.error("LLM empty content for %s", project_key)
|
|
return _llm_failure("空响应")
|
|
|
|
obj = _extract_json_object(content)
|
|
if obj is None:
|
|
log.error("LLM JSON parse failed for %s: %.200s", project_key, content)
|
|
return _llm_failure("JSON 解析失败")
|
|
|
|
summary = str(obj.get("summary_zh", "")).strip()
|
|
if not summary:
|
|
log.error("LLM empty summary_zh for %s", project_key)
|
|
return _llm_failure("summary_zh 为空")
|
|
|
|
breaking = obj.get("breaking")
|
|
if not isinstance(breaking, bool):
|
|
breaking = None
|
|
return {
|
|
"summary_zh": summary,
|
|
"breaking": breaking,
|
|
"breaking_reason": str(obj.get("breaking_reason", "")).strip() or "未发现破坏性变更",
|
|
"relevance_note": str(obj.get("relevance_note", "")).strip(),
|
|
}
|
|
except Exception as e:
|
|
log.error("LLM call failed for %s: %s", project_key, e)
|
|
return _llm_failure(str(e))
|
|
|
|
|
|
def gh_headers():
|
|
return {
|
|
"User-Agent": UA,
|
|
"Accept": "application/vnd.github+json",
|
|
}
|
|
|
|
|
|
def fetch_github_release(github_repo: str) -> tuple[dict | None, str | None]:
|
|
url = f"https://api.github.com/repos/{github_repo}/releases/latest"
|
|
try:
|
|
r = requests.get(url, headers=gh_headers(), timeout=GITHUB_TIMEOUT)
|
|
r.raise_for_status()
|
|
d = r.json()
|
|
return {
|
|
"tag_name": d.get("tag_name", ""),
|
|
"published_at": d.get("published_at", ""),
|
|
"name": d.get("name", ""),
|
|
"body": (d.get("body") or "")[:LLM_BODY_LIMIT],
|
|
}, None
|
|
except Exception as e:
|
|
log.error("GitHub release failed for %s: %s", github_repo, e)
|
|
return None, _short_reason(e)
|
|
|
|
|
|
def fetch_github_commits(github_repo: str) -> tuple[list[dict] | None, str | None]:
|
|
url = f"https://api.github.com/repos/{github_repo}/commits?per_page=5"
|
|
try:
|
|
r = requests.get(url, headers=gh_headers(), timeout=GITHUB_TIMEOUT)
|
|
r.raise_for_status()
|
|
items = []
|
|
for c in r.json():
|
|
msg = c.get("commit", {}).get("message", "")
|
|
first_line = msg.split("\n")[0][:LLM_MESSAGE_LIMIT]
|
|
date = c.get("commit", {}).get("author", {}).get("date", "")
|
|
items.append({"message": first_line, "date": date})
|
|
return items, None
|
|
except Exception as e:
|
|
log.error("GitHub commits failed for %s: %s", github_repo, e)
|
|
return None, _short_reason(e)
|
|
|
|
|
|
def fetch_npm(pkg: str) -> tuple[dict | None, str | None]:
|
|
if pkg.startswith("@"):
|
|
scope, name = pkg.split("/", 1)
|
|
encoded = f"{scope}%2F{name}"
|
|
else:
|
|
encoded = pkg
|
|
url = f"https://registry.npmjs.org/{encoded}/latest"
|
|
try:
|
|
r = requests.get(url, headers={"User-Agent": UA}, timeout=HTTP_TIMEOUT)
|
|
r.raise_for_status()
|
|
d = r.json()
|
|
return {
|
|
"version": d.get("version", ""),
|
|
"description": d.get("description", ""),
|
|
}, None
|
|
except Exception as e:
|
|
log.error("npm fetch failed for %s: %s", pkg, e)
|
|
return None, _short_reason(e)
|
|
|
|
|
|
def fetch_releases_atom(github_repo: str) -> tuple[dict | None, str | None]:
|
|
url = f"https://github.com/{github_repo}/releases.atom"
|
|
last_err: str | None = None
|
|
for attempt in range(2): # 首次 + 1 次重试 (仅 Timeout)
|
|
try:
|
|
r = requests.get(
|
|
url, headers={"User-Agent": UA}, timeout=ATOM_TIMEOUT
|
|
)
|
|
r.raise_for_status()
|
|
root = ET.fromstring(r.text)
|
|
entries = root.findall(f"{ATOM_NS}entry")
|
|
if not entries:
|
|
return None, "feed 无 entry"
|
|
entry = entries[0]
|
|
title = entry.findtext(f"{ATOM_NS}title", default="")
|
|
updated = entry.findtext(f"{ATOM_NS}updated", default="")
|
|
return {"title": title, "updated": updated}, None
|
|
except requests.exceptions.Timeout:
|
|
last_err = "超时 (github.com 限流/网络)"
|
|
log.warning(
|
|
"Atom feed timeout for %s (attempt %d/2)", github_repo, attempt + 1
|
|
)
|
|
if attempt == 0:
|
|
time.sleep(5)
|
|
except Exception as e:
|
|
last_err = _short_reason(e)
|
|
log.error("Atom feed failed for %s: %s", github_repo, e)
|
|
break
|
|
return None, last_err
|
|
|
|
|
|
def _parse_exa_text_block(text: str) -> list[dict]:
|
|
results = []
|
|
blocks = re.split(r"\n---\n", text)
|
|
for block in blocks:
|
|
block = block.strip()
|
|
if not block:
|
|
continue
|
|
title = ""
|
|
url = ""
|
|
highlights = []
|
|
in_highlights = False
|
|
for line in block.split("\n"):
|
|
if line.startswith("Title:"):
|
|
title = line[6:].strip()
|
|
in_highlights = False
|
|
elif line.startswith("URL:"):
|
|
url = line[4:].strip()
|
|
in_highlights = False
|
|
elif line.startswith("Published:") or line.startswith("Author:"):
|
|
in_highlights = False
|
|
elif line.startswith("Highlights:"):
|
|
in_highlights = True
|
|
elif in_highlights and line.strip() and not line.startswith("..."):
|
|
highlights.append(line.strip())
|
|
if title and url:
|
|
summary = " ".join(highlights)[:LLM_FIELD_LIMIT]
|
|
results.append({"title": title, "url": url, "text": summary})
|
|
return results
|
|
|
|
|
|
class ExaClient:
|
|
def __init__(self):
|
|
self.url = "https://mcp.exa.ai/mcp?tools=web_search_exa"
|
|
self.headers = {
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json, text/event-stream",
|
|
}
|
|
self.session_id: str | None = None
|
|
self._initialized = False
|
|
|
|
def initialize(self) -> bool:
|
|
try:
|
|
body = {
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "initialize",
|
|
"params": {
|
|
"protocolVersion": "2025-03-26",
|
|
"capabilities": {},
|
|
"clientInfo": {"name": "daily-watchdog", "version": "1.0"},
|
|
},
|
|
}
|
|
r = requests.post(
|
|
self.url, json=body, headers=self.headers, timeout=EXA_TIMEOUT
|
|
)
|
|
r.raise_for_status()
|
|
self.session_id = r.headers.get("Mcp-Session-Id", "")
|
|
notify = {
|
|
"jsonrpc": "2.0",
|
|
"method": "notifications/initialized",
|
|
}
|
|
h = {**self.headers}
|
|
if self.session_id:
|
|
h["Mcp-Session-Id"] = self.session_id
|
|
requests.post(
|
|
self.url, json=notify, headers=h, timeout=HTTP_TIMEOUT
|
|
)
|
|
self._initialized = True
|
|
log.info("Exa MCP initialized, session=%s", self.session_id)
|
|
return True
|
|
except Exception as e:
|
|
log.error("Exa init failed: %s", e)
|
|
return False
|
|
|
|
def search(self, query: str, num_results: int = 3) -> list[dict] | None:
|
|
if not self._initialized:
|
|
return None
|
|
try:
|
|
body = {
|
|
"jsonrpc": "2.0",
|
|
"id": 2,
|
|
"method": "tools/call",
|
|
"params": {
|
|
"name": "web_search_exa",
|
|
"arguments": {"query": query, "numResults": num_results},
|
|
},
|
|
}
|
|
h = {**self.headers}
|
|
if self.session_id:
|
|
h["Mcp-Session-Id"] = self.session_id
|
|
r = requests.post(
|
|
self.url, json=body, headers=h, timeout=EXA_TIMEOUT
|
|
)
|
|
r.raise_for_status()
|
|
text = r.text
|
|
results = []
|
|
for line in text.split("\n"):
|
|
line = line.strip()
|
|
if not line.startswith("data:"):
|
|
continue
|
|
payload = line[5:].strip()
|
|
if not payload:
|
|
continue
|
|
try:
|
|
obj = json.loads(payload)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if "result" not in obj:
|
|
continue
|
|
content = obj["result"].get("content", [])
|
|
for c in content:
|
|
raw = c.get("text", "")
|
|
parsed = _parse_exa_text_block(raw)
|
|
results.extend(parsed)
|
|
return results if results else None
|
|
except Exception as e:
|
|
log.error("Exa search failed for '%s': %s", query, e)
|
|
return None
|
|
|
|
|
|
def check_npm_relevance(pkg: str, npm_data: dict) -> bool:
|
|
desc = (npm_data.get("description") or "").lower()
|
|
keywords = [
|
|
"vectorize",
|
|
"hindsight",
|
|
"replay",
|
|
"debugging",
|
|
"browser",
|
|
"record",
|
|
"session",
|
|
]
|
|
return any(kw in desc for kw in keywords)
|
|
|
|
|
|
def _collect_one_project(proj: dict, exa: ExaClient | None) -> dict:
|
|
"""Fetch all sources for one project and summarize with the LLM.
|
|
|
|
Runs inside a worker thread. Every fetcher already catches its own
|
|
exceptions, so one failing source (or a failing LLM call) never raises
|
|
out of here and never affects other projects. `exa` is None when the
|
|
shared MCP session failed to initialize.
|
|
"""
|
|
log.info("Collecting data for %s", proj["label"])
|
|
data: dict = {
|
|
"key": proj["key"],
|
|
"label": proj["label"],
|
|
"star": proj["star"],
|
|
"installed": proj["installed"],
|
|
"github_release": None,
|
|
"github_commits": None,
|
|
"npm": None,
|
|
"atom": None,
|
|
"exa_results": None,
|
|
"source_errors": [],
|
|
"sources_used": [],
|
|
"llm_summary": None,
|
|
}
|
|
|
|
rel, rel_err = fetch_github_release(proj["github"])
|
|
if rel:
|
|
data["github_release"] = rel
|
|
data["sources_used"].append("GitHub Releases API")
|
|
else:
|
|
data["source_errors"].append(
|
|
f"GitHub Releases API: {rel_err or '请求失败'}"
|
|
)
|
|
|
|
commits, commits_err = fetch_github_commits(proj["github"])
|
|
if commits:
|
|
data["github_commits"] = commits
|
|
data["sources_used"].append("GitHub commits API")
|
|
else:
|
|
data["source_errors"].append(
|
|
f"GitHub commits API: {commits_err or '请求失败'}"
|
|
)
|
|
|
|
npm_data, npm_err = fetch_npm(proj["npm"])
|
|
if npm_data:
|
|
if proj.get("npm_relevance_check"):
|
|
if check_npm_relevance(proj["npm"], npm_data):
|
|
data["npm"] = npm_data
|
|
data["sources_used"].append("npm registry")
|
|
else:
|
|
log.info(
|
|
"npm package '%s' not relevant, skipping", proj["npm"]
|
|
)
|
|
data["source_errors"].append(
|
|
"npm registry: 包存在但与本 repo 不相关, 已跳过"
|
|
)
|
|
else:
|
|
data["npm"] = npm_data
|
|
data["sources_used"].append("npm registry")
|
|
else:
|
|
data["source_errors"].append(f"npm registry: {npm_err or '请求失败'}")
|
|
|
|
atom, atom_err = fetch_releases_atom(proj["github"])
|
|
if atom:
|
|
data["atom"] = atom
|
|
data["sources_used"].append("GitHub releases.atom")
|
|
else:
|
|
data["source_errors"].append(
|
|
f"GitHub releases.atom: {atom_err or '请求失败'}"
|
|
)
|
|
|
|
if exa is not None:
|
|
query = f"{proj['label']} latest release changelog"
|
|
exa_res = exa.search(query)
|
|
if exa_res:
|
|
data["exa_results"] = exa_res
|
|
data["sources_used"].append("Exa web search")
|
|
else:
|
|
data["source_errors"].append("Exa web search: 无结果或查询失败")
|
|
else:
|
|
data["source_errors"].append("Exa web search: MCP 初始化失败")
|
|
|
|
collected_for_llm = {
|
|
"github_release": data["github_release"],
|
|
"github_commits": data["github_commits"],
|
|
"npm": data["npm"],
|
|
"atom": data["atom"],
|
|
"exa_results": data["exa_results"],
|
|
}
|
|
data["llm_summary"] = summarize_with_llm(proj["key"], collected_for_llm)
|
|
|
|
return data
|
|
|
|
|
|
def collect_all() -> tuple[list[dict], dict[str, int], list[str]]:
|
|
exa = ExaClient()
|
|
exa_ok = exa.initialize()
|
|
if not exa_ok:
|
|
log.warning("Exa MCP init failed — all projects will note it")
|
|
|
|
# I/O bound (HTTP + LLM), so threads sidestep the GIL. One hanging
|
|
# project cannot stall the run: its future is abandoned after
|
|
# COLLECT_PROJECT_TIMEOUT and the thread is left to die on its own.
|
|
with futures.ThreadPoolExecutor(max_workers=COLLECT_MAX_WORKERS) as pool:
|
|
future_by_index = {
|
|
i: pool.submit(_collect_one_project, proj, exa if exa_ok else None)
|
|
for i, proj in enumerate(PROJECTS)
|
|
}
|
|
results_by_index: dict[int, dict] = {}
|
|
for i, fut in future_by_index.items():
|
|
proj = PROJECTS[i]
|
|
try:
|
|
results_by_index[i] = fut.result(timeout=COLLECT_PROJECT_TIMEOUT)
|
|
except Exception as e:
|
|
log.error(
|
|
"collect timed out or crashed for %s: %s", proj["label"], e
|
|
)
|
|
results_by_index[i] = {
|
|
"key": proj["key"],
|
|
"label": proj["label"],
|
|
"star": proj["star"],
|
|
"installed": proj["installed"],
|
|
"github_release": None,
|
|
"github_commits": None,
|
|
"npm": None,
|
|
"atom": None,
|
|
"exa_results": None,
|
|
"source_errors": [f"并行抓取超时或异常: {_short_reason(e)}"],
|
|
"sources_used": [],
|
|
"llm_summary": None,
|
|
}
|
|
|
|
# Preserve PROJECTS order; aggregate per-source stats from the results
|
|
# produced by the worker threads (counts are merged here, single-threaded).
|
|
project_results = [results_by_index[i] for i in range(len(PROJECTS))]
|
|
source_counts: dict[str, int] = {s: 0 for s in CANONICAL_SOURCES}
|
|
sources_failed: set[str] = set()
|
|
if not exa_ok:
|
|
sources_failed.add("Exa web search")
|
|
for data in project_results:
|
|
for src in data["sources_used"]:
|
|
if src in source_counts:
|
|
source_counts[src] += 1
|
|
for err in data["source_errors"]:
|
|
src, _sep, _reason = err.partition(": ")
|
|
if src in source_counts:
|
|
sources_failed.add(src)
|
|
|
|
return project_results, source_counts, sorted(sources_failed)
|
|
|
|
|
|
def determine_project_status(data: dict) -> str:
|
|
has_any = any(
|
|
[
|
|
data["github_release"],
|
|
data["github_commits"],
|
|
data["npm"],
|
|
data["atom"],
|
|
]
|
|
)
|
|
if has_any and not data["source_errors"]:
|
|
return "ok"
|
|
if has_any:
|
|
return "partial"
|
|
return "failed"
|
|
|
|
|
|
def _split_source_error(err: str) -> tuple[str, str]:
|
|
src, sep, reason = err.partition(": ")
|
|
if not sep:
|
|
return err, "未知原因"
|
|
return src, reason
|
|
|
|
|
|
def render_report(
|
|
date_str: str,
|
|
project_results: list[dict],
|
|
source_counts: dict[str, int],
|
|
) -> str:
|
|
lines = []
|
|
lines.append(f"# AI Agent 升级迭代日报 - {date_str}")
|
|
lines.append("")
|
|
lines.append("## 数据源")
|
|
lines.append("")
|
|
total = len(PROJECTS)
|
|
for src in CANONICAL_SOURCES:
|
|
n = source_counts.get(src, 0)
|
|
lines.append(f"- {src} ({n}/{total})")
|
|
lines.append("")
|
|
|
|
all_failed = all(determine_project_status(d) == "failed" for d in project_results)
|
|
if all_failed:
|
|
lines.append("> ⚠️ 所有项目全部数据源抓取失败")
|
|
lines.append("")
|
|
|
|
for data in project_results:
|
|
status = determine_project_status(data)
|
|
star_mark = " ★" if data["star"] else ""
|
|
install_mark = f" (已装 {data['installed']})" if data["installed"] else " (未装)"
|
|
lines.append(f"## {data['label']}{star_mark}{install_mark}")
|
|
lines.append("")
|
|
|
|
if status == "failed":
|
|
lines.append("⚠️ 全部数据源抓取失败:")
|
|
lines.append("")
|
|
for err in data["source_errors"]:
|
|
src, reason = _split_source_error(err)
|
|
lines.append(f"⚠️ 抓取失败: {src} — {reason}")
|
|
lines.append("")
|
|
continue
|
|
|
|
if data["github_release"]:
|
|
rel = data["github_release"]
|
|
lines.append(f"**Latest Release:** `{rel['tag_name']}`")
|
|
lines.append(f"**发布日期:** {rel['published_at']}")
|
|
if rel["name"]:
|
|
lines.append(f"**Release 名称:** {rel['name']}")
|
|
lines.append("")
|
|
if rel["body"]:
|
|
lines.append("**Changelog 摘要:**")
|
|
lines.append(f"> {rel['body'][:LLM_FIELD_LIMIT]}")
|
|
lines.append("")
|
|
elif data["github_commits"]:
|
|
lines.append("**最近 Commits (无 release):**")
|
|
for c in data["github_commits"]:
|
|
lines.append(f"- [{c['date'][:10]}] {c['message']}")
|
|
lines.append("")
|
|
else:
|
|
lines.append("⚠️ GitHub 数据不可用")
|
|
for err in data["source_errors"]:
|
|
if err.startswith("GitHub Releases API:") or err.startswith(
|
|
"GitHub commits API:"
|
|
):
|
|
src, reason = _split_source_error(err)
|
|
lines.append(f"⚠️ 抓取失败: {src} — {reason}")
|
|
lines.append("")
|
|
|
|
llm = data["llm_summary"] or {}
|
|
if llm.get("summary_zh"):
|
|
lines.append(f"**LLM 总结:** {llm['summary_zh']}")
|
|
lines.append("")
|
|
lines.append(
|
|
f"**Breaking Changes:** {llm.get('breaking_reason', 'LLM 总结失败')}"
|
|
)
|
|
lines.append("")
|
|
|
|
if data["npm"]:
|
|
npm = data["npm"]
|
|
lines.append(f"**npm latest:** `{npm['version']}` — {npm['description']}")
|
|
lines.append("")
|
|
elif any(
|
|
e.startswith("npm registry: 包存在但与本 repo 不相关")
|
|
for e in data["source_errors"]
|
|
):
|
|
lines.append("**npm:** ⚠️ 包存在但与本 repo 不相关, 已跳过")
|
|
lines.append("")
|
|
elif any(e.startswith("npm registry:") for e in data["source_errors"]):
|
|
err = next(
|
|
e for e in data["source_errors"] if e.startswith("npm registry:")
|
|
)
|
|
src, reason = _split_source_error(err)
|
|
lines.append(f"**npm:** ⚠️ 抓取失败: {src} — {reason}")
|
|
lines.append("")
|
|
|
|
if data["atom"]:
|
|
atom = data["atom"]
|
|
lines.append(f"**releases.atom 最新:** {atom['title']} ({atom['updated']})")
|
|
lines.append("")
|
|
elif any(
|
|
e.startswith("GitHub releases.atom:") for e in data["source_errors"]
|
|
):
|
|
err = next(
|
|
e
|
|
for e in data["source_errors"]
|
|
if e.startswith("GitHub releases.atom:")
|
|
)
|
|
src, reason = _split_source_error(err)
|
|
lines.append(f"**releases.atom:** ⚠️ 抓取失败: {src} — {reason}")
|
|
lines.append("")
|
|
|
|
llm = data["llm_summary"] or {}
|
|
relevance = llm.get("relevance_note", "")
|
|
if relevance:
|
|
lines.append(f"★ **相关性:** {relevance}")
|
|
lines.append("")
|
|
else:
|
|
lines.append("★ **相关性:** LLM 总结失败, 相关性待评估")
|
|
lines.append("")
|
|
|
|
if data["exa_results"]:
|
|
lines.append("**Exa 搜索结果:**")
|
|
lines.append("")
|
|
for item in data["exa_results"][:3]:
|
|
title = item.get("title", "无标题")
|
|
url = item.get("url", "")
|
|
text = item.get("text", "")[:LLM_MESSAGE_LIMIT]
|
|
lines.append(f"> [{title}]({url}): {text}")
|
|
lines.append("")
|
|
elif any(e.startswith("Exa web search:") for e in data["source_errors"]):
|
|
err = next(
|
|
e for e in data["source_errors"] if e.startswith("Exa web search:")
|
|
)
|
|
src, reason = _split_source_error(err)
|
|
lines.append(f"**Exa 搜索:** ⚠️ 抓取失败: {src} — {reason}")
|
|
lines.append("")
|
|
|
|
lines.append("---")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def save_to_sqlite(date_str: str, markdown: str, meta: dict) -> None:
|
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(str(DB_PATH))
|
|
try:
|
|
conn.execute(
|
|
"""CREATE TABLE IF NOT EXISTS reports (
|
|
date TEXT PRIMARY KEY,
|
|
markdown TEXT NOT NULL,
|
|
meta_json TEXT NOT NULL,
|
|
created_at TEXT NOT NULL
|
|
)"""
|
|
)
|
|
now = datetime.now().isoformat()
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO reports (date, markdown, meta_json, created_at) VALUES (?, ?, ?, ?)",
|
|
(date_str, markdown, json.dumps(meta, ensure_ascii=False), now),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def save_markdown(date_str: str, markdown: str) -> Path:
|
|
LOGS_DIR.mkdir(parents=True, exist_ok=True)
|
|
path = LOGS_DIR / f"daily-agent-watchdog-{date_str}.md"
|
|
path.write_text(markdown, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def _watchdog_already_running() -> bool:
|
|
try:
|
|
out = subprocess.check_output(
|
|
["pgrep", "-f", "daily_watchdog.py"], text=True, timeout=10
|
|
)
|
|
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError):
|
|
return False
|
|
own = str(os.getpid())
|
|
for pid in out.split():
|
|
pid = pid.strip()
|
|
if not pid or pid == own:
|
|
continue
|
|
# 只认真正的 python 脚本进程, 排除命令行里恰好提到该文件名的进程
|
|
try:
|
|
with open(f"/proc/{pid}/cmdline", "rb") as f:
|
|
argv = f.read().split(b"\x00")
|
|
except OSError:
|
|
continue
|
|
if not argv or not argv[0]:
|
|
continue
|
|
exe = os.path.basename(argv[0].decode(errors="replace"))
|
|
if exe.startswith("python"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _today_report_ok(date_str: str) -> bool:
|
|
if not DB_PATH.exists():
|
|
return False
|
|
try:
|
|
conn = sqlite3.connect(str(DB_PATH))
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT meta_json FROM reports WHERE date = ?", (date_str,)
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
except Exception as e:
|
|
log.warning("fix-if-missing check failed: %s", e)
|
|
return False
|
|
if not row:
|
|
return False
|
|
try:
|
|
meta = json.loads(row[0])
|
|
except (json.JSONDecodeError, TypeError):
|
|
return False
|
|
projects = meta.get("projects", {})
|
|
ok_count = sum(
|
|
1
|
|
for st in projects.values()
|
|
if isinstance(st, dict) and st.get("status") != "failed"
|
|
)
|
|
return ok_count >= 2
|
|
|
|
|
|
def main() -> int:
|
|
fix_if_missing = "--fix-if-missing" in sys.argv[1:]
|
|
date_str = datetime.now().strftime("%Y-%m-%d")
|
|
|
|
if _watchdog_already_running():
|
|
log.warning("Another daily_watchdog instance is already running — skip")
|
|
return 0
|
|
|
|
if fix_if_missing and _today_report_ok(date_str):
|
|
log.info("today's report already OK — skip full run (--fix-if-missing)")
|
|
return 0
|
|
|
|
log.info("Starting daily watchdog for %s", date_str)
|
|
|
|
project_results, source_counts, sources_failed = collect_all()
|
|
|
|
all_failed = all(determine_project_status(d) == "failed" for d in project_results)
|
|
|
|
markdown = render_report(date_str, project_results, source_counts)
|
|
|
|
meta = {
|
|
"sources_ok": sorted(k for k, v in source_counts.items() if v > 0),
|
|
"sources_failed": sources_failed,
|
|
"source_counts": source_counts,
|
|
"projects": {
|
|
d["key"]: {
|
|
"status": determine_project_status(d),
|
|
"sources_used": d["sources_used"],
|
|
}
|
|
for d in project_results
|
|
},
|
|
}
|
|
|
|
save_to_sqlite(date_str, markdown, meta)
|
|
md_path = save_markdown(date_str, markdown)
|
|
log.info("Report saved to %s", md_path)
|
|
log.info("SQLite record saved to %s", DB_PATH)
|
|
|
|
if all_failed:
|
|
log.error("All projects failed — exit code 1")
|
|
return 1
|
|
|
|
log.info("Daily watchdog completed successfully")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|