347 lines
14 KiB
Python
347 lines
14 KiB
Python
"""Analysis service: context building, LLM prompt assembly, SSE event framing.
|
|
|
|
Pure functions — no Flask imports. All DB access goes through an explicit
|
|
sqlite3 connection argument (row_factory=sqlite3.Row expected), so this
|
|
module can also be imported by scripts/daily_watchdog.py without an app
|
|
context. Error values are plain `(dict, status)` tuples; routes jsonify them.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import time
|
|
|
|
import requests as http_requests
|
|
|
|
import config
|
|
from repositories import issue_repo, project_repo, release_repo
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DELIM_REASONING = "===REASONING==="
|
|
DELIM_CONCLUSION = "===CONCLUSION==="
|
|
|
|
PRESET_MAP = {
|
|
"activity": "请基于最近 release + issue 评估项目活跃度 (1-10 分) 并给出证据",
|
|
"evolution": "请按时间列 5 个关键 release 的变更重点, 给出演进方向判断",
|
|
"issues_hot": "请把 issue 按主题聚类, 列 top 3 热点, 每条给 1 句描述",
|
|
}
|
|
|
|
PRESET_LIST = [
|
|
{"id": "activity", "label": "项目活跃度评估", "icon": "📊", "description": "基于最近 release + issue 评估项目活跃度 (1-10) + 证据"},
|
|
{"id": "evolution", "label": "版本演进分析", "icon": "🚀", "description": "按时间列 5 个关键 release 的变更重点, 给演进方向判断"},
|
|
{"id": "issues_hot", "label": "Issue 热点聚类", "icon": "🔥", "description": "把 issue 按主题聚类, 列 top 3 热点 + 每条 1 句描述"},
|
|
]
|
|
|
|
SYSTEM_PROMPT = "你是 AI 项目分析助手, 基于以下数据进行分析。请用中文回答。"
|
|
|
|
# Prompt trimming budgets live in config (env-overridable, shared with
|
|
# scripts/daily_watchdog.py). Re-exported for convenient imports/tests.
|
|
LLM_BODY_LIMIT = config.LLM_BODY_LIMIT
|
|
LLM_FIELD_LIMIT = config.LLM_FIELD_LIMIT
|
|
LLM_MESSAGE_LIMIT = config.LLM_MESSAGE_LIMIT
|
|
LLM_TOTAL_LIMIT = config.LLM_TOTAL_LIMIT
|
|
|
|
FORMAT_REQ = (
|
|
"要求: 严格按以下格式输出, 不要输出其他任何标记:\n"
|
|
"===REASONING===\n"
|
|
"(这里写你的分析推理过程: 数据观察、初步判断、深入推理)\n"
|
|
"===CONCLUSION===\n"
|
|
"(这里写最终结论: 先给一句总结论, 再用 2-3 句概括关键依据, 不少于 50 字)"
|
|
)
|
|
|
|
|
|
def get_presets():
|
|
return {"presets": PRESET_LIST}
|
|
|
|
|
|
def resolve_question(question: str, preset: str) -> str:
|
|
"""If only a preset id is given, map it to the canonical question text."""
|
|
if not question and preset:
|
|
return PRESET_MAP.get(preset, preset)
|
|
return question
|
|
|
|
|
|
def build_context(scope: str, db_conn):
|
|
"""Build the context dict for a scope. Single source of truth used by
|
|
/api/context/<scope> and /api/analyze/stream.
|
|
|
|
Returns (result_dict_or_None, error_or_None). The error is a plain
|
|
(payload_dict, status_code) tuple — the route jsonifies it.
|
|
"""
|
|
result = {
|
|
"scope_type": scope,
|
|
"project_id": None,
|
|
"data_summary": "",
|
|
"releases": [],
|
|
"issues": [],
|
|
"stats": {},
|
|
}
|
|
|
|
if scope == "home":
|
|
stats = project_repo.get_stats(db_conn)
|
|
result["stats"] = stats
|
|
result["data_summary"] = (
|
|
f"共 {stats['total_projects']} 个项目, {stats['total_releases']} 个 release, "
|
|
f"{stats['total_issues']} 个 issue, {stats['total_stars']} stars"
|
|
)
|
|
result["releases"] = release_repo.list_recent_releases_slim(db_conn, limit=5)
|
|
result["issues"] = issue_repo.list_recent_issues(db_conn, limit=10)
|
|
elif scope.startswith("project/"):
|
|
try:
|
|
pid = int(scope.split("/")[1])
|
|
except (IndexError, ValueError):
|
|
return None, ({"error": "invalid project scope"}, 400)
|
|
result["project_id"] = pid
|
|
proj = project_repo.get_project(db_conn, pid)
|
|
if proj is None:
|
|
return None, ({"error": "project not found"}, 404)
|
|
result["data_summary"] = (
|
|
f"项目: {proj['name']} ({proj.get('full_name', '')}), {proj.get('stars', 0)} stars"
|
|
)
|
|
result["releases"] = release_repo.list_releases_for_project_slim(db_conn, pid, limit=5)
|
|
result["issues"] = issue_repo.list_issues_for_project(db_conn, pid, limit=10)
|
|
elif scope == "projects":
|
|
projects = project_repo.list_project_summaries(db_conn)
|
|
result["data_summary"] = f"共 {len(projects)} 个项目"
|
|
result["releases"] = release_repo.list_recent_releases_slim(db_conn, limit=5)
|
|
result["issues"] = issue_repo.list_recent_issues(db_conn, limit=10)
|
|
elif scope.startswith("search/"):
|
|
query = scope[7:]
|
|
result["data_summary"] = f"搜索: {query}"
|
|
result["releases"] = release_repo.search_releases_slim(db_conn, query, limit=5)
|
|
result["issues"] = issue_repo.search_issues(db_conn, query, limit=10)
|
|
else:
|
|
return None, ({"error": "unknown scope"}, 400)
|
|
|
|
return result, None
|
|
|
|
|
|
def _truncate(text, limit):
|
|
"""Truncate a string to limit chars, appending '...' when cut."""
|
|
if not isinstance(text, str):
|
|
return text
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[:limit] + "..."
|
|
|
|
|
|
def _slim_for_llm(data):
|
|
"""Return a trimmed copy of the context dict for the LLM prompt.
|
|
|
|
Keeps only the fields the model needs and truncates long bodies so the
|
|
serialized JSON stays small enough for fast prefill.
|
|
"""
|
|
slim = dict(data)
|
|
|
|
if isinstance(data.get("releases"), list):
|
|
slim_releases = []
|
|
for rel in data["releases"]:
|
|
if not isinstance(rel, dict):
|
|
slim_releases.append(rel)
|
|
continue
|
|
slim_releases.append({
|
|
"tag_name": rel.get("tag_name"),
|
|
"name": rel.get("name"),
|
|
"published_at": rel.get("published_at"),
|
|
"body": _truncate(rel.get("body", ""), LLM_BODY_LIMIT),
|
|
})
|
|
slim["releases"] = slim_releases
|
|
|
|
if isinstance(data.get("issues"), list):
|
|
slim_issues = []
|
|
for issue in data["issues"]:
|
|
if not isinstance(issue, dict):
|
|
slim_issues.append(issue)
|
|
continue
|
|
item = {
|
|
"title": issue.get("title"),
|
|
"state": issue.get("state"),
|
|
"created_at": issue.get("created_at"),
|
|
}
|
|
if "body" in issue:
|
|
item["body"] = _truncate(issue.get("body") or "", LLM_FIELD_LIMIT)
|
|
slim_issues.append(item)
|
|
slim["issues"] = slim_issues
|
|
|
|
if isinstance(data.get("messages"), list):
|
|
slim["messages"] = [
|
|
_truncate(m, LLM_MESSAGE_LIMIT) if isinstance(m, str)
|
|
else {**m, "content": _truncate(m.get("content", ""), LLM_MESSAGE_LIMIT)} if isinstance(m, dict)
|
|
else m
|
|
for m in data["messages"]
|
|
]
|
|
|
|
serialized = json.dumps(slim, ensure_ascii=False, default=str)
|
|
if len(serialized) > LLM_TOTAL_LIMIT:
|
|
if isinstance(slim.get("releases"), list):
|
|
slim["releases"] = slim["releases"][:3]
|
|
if isinstance(slim.get("issues"), list):
|
|
slim["issues"] = slim["issues"][:5]
|
|
logger.info(
|
|
"_slim_for_llm: payload still >%d chars, cut to %d releases / %d issues",
|
|
LLM_TOTAL_LIMIT,
|
|
len(slim.get("releases", [])),
|
|
len(slim.get("issues", [])),
|
|
)
|
|
return slim
|
|
|
|
|
|
def build_prompts(data, question):
|
|
"""Assemble system/user messages + the human-readable prompt display."""
|
|
slim_data = _slim_for_llm(data)
|
|
data_json = json.dumps(slim_data, ensure_ascii=False, default=str)
|
|
user_prompt = f"分析数据: {data_json}\n\n用户问题: {question}\n\n{FORMAT_REQ}"
|
|
messages = [
|
|
{"role": "system", "content": SYSTEM_PROMPT},
|
|
{"role": "user", "content": user_prompt},
|
|
]
|
|
full_prompt_display = f"[SYSTEM]\n{SYSTEM_PROMPT}\n\n[USER]\n{user_prompt}"
|
|
return messages, full_prompt_display
|
|
|
|
|
|
def _sse(event, payload):
|
|
return f"event: {event}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
|
|
|
|
|
def stream_analysis_events(data, question):
|
|
"""Generator yielding SSE frames: step/token/section_start/error/complete.
|
|
|
|
Pure of Flask: safe to consume after the request context has closed,
|
|
as long as `data` and `question` were prepared inside the view.
|
|
"""
|
|
start_time = time.time()
|
|
data_summary = data["data_summary"]
|
|
messages, full_prompt_display = build_prompts(data, question)
|
|
|
|
yield _sse("step", {"step": "context", "content": f"已收集上下文: {data_summary}"})
|
|
yield _sse("step", {"step": "prompt", "content": full_prompt_display})
|
|
|
|
def emit_token(t):
|
|
return _sse("token", {"text": t})
|
|
|
|
try:
|
|
resp = http_requests.post(
|
|
f"{config.LLM_URL}/chat/completions",
|
|
json={"model": config.LLM_MODEL, "messages": messages, "stream": True, "max_tokens": 1024},
|
|
stream=True,
|
|
timeout=config.LLM_TIMEOUT,
|
|
)
|
|
resp.raise_for_status()
|
|
|
|
last_keepalive = time.time()
|
|
holdback = max(len(DELIM_REASONING), len(DELIM_CONCLUSION)) - 1
|
|
pending = ""
|
|
reasoning_started = False
|
|
conclusion_started = False
|
|
|
|
for line in resp.iter_lines():
|
|
if not line:
|
|
continue
|
|
line = line.decode("utf-8") if isinstance(line, bytes) else line
|
|
|
|
now = time.time()
|
|
if now - last_keepalive > 15:
|
|
yield ": keepalive\n\n"
|
|
last_keepalive = now
|
|
|
|
if line.startswith("data: "):
|
|
chunk_data = line[6:]
|
|
if chunk_data.strip() == "[DONE]":
|
|
break
|
|
try:
|
|
chunk = json.loads(chunk_data)
|
|
delta = chunk.get("choices", [{}])[0].get("delta", {})
|
|
token_text = delta.get("content", "")
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if not token_text:
|
|
continue
|
|
|
|
pending += token_text
|
|
|
|
# 滚动 buffer: 分隔符可能跨 chunk, 保留尾部前缀再下发
|
|
while pending:
|
|
idx_r = pending.find(DELIM_REASONING)
|
|
idx_c = pending.find(DELIM_CONCLUSION)
|
|
idx = -1
|
|
delim = None
|
|
if idx_r != -1 and (idx_c == -1 or idx_r < idx_c):
|
|
idx, delim = idx_r, DELIM_REASONING
|
|
elif idx_c != -1:
|
|
idx, delim = idx_c, DELIM_CONCLUSION
|
|
|
|
if idx != -1:
|
|
if idx > 0:
|
|
if not reasoning_started:
|
|
reasoning_started = True
|
|
yield "event: reasoning_start\ndata: {}\n\n"
|
|
yield emit_token(pending[:idx])
|
|
pending = pending[idx + len(delim):]
|
|
if delim == DELIM_REASONING:
|
|
if not reasoning_started:
|
|
reasoning_started = True
|
|
yield "event: reasoning_start\ndata: {}\n\n"
|
|
else:
|
|
if not reasoning_started:
|
|
reasoning_started = True
|
|
yield "event: reasoning_start\ndata: {}\n\n"
|
|
if not conclusion_started:
|
|
conclusion_started = True
|
|
yield "event: conclusion_start\ndata: {}\n\n"
|
|
continue
|
|
|
|
keep = 0
|
|
tail = pending[-holdback:] if len(pending) > holdback else pending
|
|
for d in (DELIM_REASONING, DELIM_CONCLUSION):
|
|
for k in range(min(len(tail), len(d) - 1), 0, -1):
|
|
if tail.endswith(d[:k]):
|
|
keep = max(keep, k)
|
|
break
|
|
safe = pending[:len(pending) - keep] if keep else pending
|
|
pending = pending[len(safe):]
|
|
if not safe:
|
|
break
|
|
if not reasoning_started:
|
|
reasoning_started = True
|
|
yield "event: reasoning_start\ndata: {}\n\n"
|
|
yield emit_token(safe)
|
|
|
|
if pending:
|
|
if not reasoning_started:
|
|
reasoning_started = True
|
|
yield "event: reasoning_start\ndata: {}\n\n"
|
|
yield emit_token(pending)
|
|
|
|
except Exception as e:
|
|
yield _sse("error", {"error": f"LLM call failed: {str(e)}"})
|
|
return
|
|
|
|
# 防御: 模型没给分隔符时补发 section_start, 保证前端 tab 一定有入口
|
|
if not reasoning_started:
|
|
yield "event: reasoning_start\ndata: {}\n\n"
|
|
if not conclusion_started:
|
|
yield "event: conclusion_start\ndata: {}\n\n"
|
|
|
|
elapsed = round(time.time() - start_time, 1)
|
|
yield _sse("step", {"step": "complete", "content": f"分析完成, 总耗时 {elapsed} 秒"})
|
|
|
|
|
|
def summarize_with_llm(messages, model=None, timeout=None):
|
|
"""Non-streaming LLM call returning the assistant content string.
|
|
|
|
Kept Flask-free so scripts/daily_watchdog.py (and future cron jobs) can
|
|
reuse the same endpoint configuration.
|
|
"""
|
|
resp = http_requests.post(
|
|
f"{config.LLM_URL}/chat/completions",
|
|
json={
|
|
"model": model or config.LLM_MODEL,
|
|
"messages": messages,
|
|
"stream": False,
|
|
"max_tokens": 1024,
|
|
},
|
|
timeout=timeout or config.LLM_TIMEOUT,
|
|
)
|
|
resp.raise_for_status()
|
|
return resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
|