initial: blog-app snapshot
This commit is contained in:
omo
346
services/analysis.py
Normal file
346
services/analysis.py
Normal file
@@ -0,0 +1,346 @@
|
||||
"""Analysis service: context building, LLM prompt assembly, SSE event framing.
|
||||
|
||||
Pure functions — no Flask imports. All DB access goes through an explicit
|
||||
sqlite3 connection argument (row_factory=sqlite3.Row expected), so this
|
||||
module can also be imported by scripts/daily_watchdog.py without an app
|
||||
context. Error values are plain `(dict, status)` tuples; routes jsonify them.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
import requests as http_requests
|
||||
|
||||
import config
|
||||
from repositories import issue_repo, project_repo, release_repo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DELIM_REASONING = "===REASONING==="
|
||||
DELIM_CONCLUSION = "===CONCLUSION==="
|
||||
|
||||
PRESET_MAP = {
|
||||
"activity": "请基于最近 release + issue 评估项目活跃度 (1-10 分) 并给出证据",
|
||||
"evolution": "请按时间列 5 个关键 release 的变更重点, 给出演进方向判断",
|
||||
"issues_hot": "请把 issue 按主题聚类, 列 top 3 热点, 每条给 1 句描述",
|
||||
}
|
||||
|
||||
PRESET_LIST = [
|
||||
{"id": "activity", "label": "项目活跃度评估", "icon": "📊", "description": "基于最近 release + issue 评估项目活跃度 (1-10) + 证据"},
|
||||
{"id": "evolution", "label": "版本演进分析", "icon": "🚀", "description": "按时间列 5 个关键 release 的变更重点, 给演进方向判断"},
|
||||
{"id": "issues_hot", "label": "Issue 热点聚类", "icon": "🔥", "description": "把 issue 按主题聚类, 列 top 3 热点 + 每条 1 句描述"},
|
||||
]
|
||||
|
||||
SYSTEM_PROMPT = "你是 AI 项目分析助手, 基于以下数据进行分析。请用中文回答。"
|
||||
|
||||
# Prompt trimming budgets live in config (env-overridable, shared with
|
||||
# scripts/daily_watchdog.py). Re-exported for convenient imports/tests.
|
||||
LLM_BODY_LIMIT = config.LLM_BODY_LIMIT
|
||||
LLM_FIELD_LIMIT = config.LLM_FIELD_LIMIT
|
||||
LLM_MESSAGE_LIMIT = config.LLM_MESSAGE_LIMIT
|
||||
LLM_TOTAL_LIMIT = config.LLM_TOTAL_LIMIT
|
||||
|
||||
FORMAT_REQ = (
|
||||
"要求: 严格按以下格式输出, 不要输出其他任何标记:\n"
|
||||
"===REASONING===\n"
|
||||
"(这里写你的分析推理过程: 数据观察、初步判断、深入推理)\n"
|
||||
"===CONCLUSION===\n"
|
||||
"(这里写最终结论: 先给一句总结论, 再用 2-3 句概括关键依据, 不少于 50 字)"
|
||||
)
|
||||
|
||||
|
||||
def get_presets():
|
||||
return {"presets": PRESET_LIST}
|
||||
|
||||
|
||||
def resolve_question(question: str, preset: str) -> str:
|
||||
"""If only a preset id is given, map it to the canonical question text."""
|
||||
if not question and preset:
|
||||
return PRESET_MAP.get(preset, preset)
|
||||
return question
|
||||
|
||||
|
||||
def build_context(scope: str, db_conn):
|
||||
"""Build the context dict for a scope. Single source of truth used by
|
||||
/api/context/<scope> and /api/analyze/stream.
|
||||
|
||||
Returns (result_dict_or_None, error_or_None). The error is a plain
|
||||
(payload_dict, status_code) tuple — the route jsonifies it.
|
||||
"""
|
||||
result = {
|
||||
"scope_type": scope,
|
||||
"project_id": None,
|
||||
"data_summary": "",
|
||||
"releases": [],
|
||||
"issues": [],
|
||||
"stats": {},
|
||||
}
|
||||
|
||||
if scope == "home":
|
||||
stats = project_repo.get_stats(db_conn)
|
||||
result["stats"] = stats
|
||||
result["data_summary"] = (
|
||||
f"共 {stats['total_projects']} 个项目, {stats['total_releases']} 个 release, "
|
||||
f"{stats['total_issues']} 个 issue, {stats['total_stars']} stars"
|
||||
)
|
||||
result["releases"] = release_repo.list_recent_releases_slim(db_conn, limit=5)
|
||||
result["issues"] = issue_repo.list_recent_issues(db_conn, limit=10)
|
||||
elif scope.startswith("project/"):
|
||||
try:
|
||||
pid = int(scope.split("/")[1])
|
||||
except (IndexError, ValueError):
|
||||
return None, ({"error": "invalid project scope"}, 400)
|
||||
result["project_id"] = pid
|
||||
proj = project_repo.get_project(db_conn, pid)
|
||||
if proj is None:
|
||||
return None, ({"error": "project not found"}, 404)
|
||||
result["data_summary"] = (
|
||||
f"项目: {proj['name']} ({proj.get('full_name', '')}), {proj.get('stars', 0)} stars"
|
||||
)
|
||||
result["releases"] = release_repo.list_releases_for_project_slim(db_conn, pid, limit=5)
|
||||
result["issues"] = issue_repo.list_issues_for_project(db_conn, pid, limit=10)
|
||||
elif scope == "projects":
|
||||
projects = project_repo.list_project_summaries(db_conn)
|
||||
result["data_summary"] = f"共 {len(projects)} 个项目"
|
||||
result["releases"] = release_repo.list_recent_releases_slim(db_conn, limit=5)
|
||||
result["issues"] = issue_repo.list_recent_issues(db_conn, limit=10)
|
||||
elif scope.startswith("search/"):
|
||||
query = scope[7:]
|
||||
result["data_summary"] = f"搜索: {query}"
|
||||
result["releases"] = release_repo.search_releases_slim(db_conn, query, limit=5)
|
||||
result["issues"] = issue_repo.search_issues(db_conn, query, limit=10)
|
||||
else:
|
||||
return None, ({"error": "unknown scope"}, 400)
|
||||
|
||||
return result, None
|
||||
|
||||
|
||||
def _truncate(text, limit):
|
||||
"""Truncate a string to limit chars, appending '...' when cut."""
|
||||
if not isinstance(text, str):
|
||||
return text
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[:limit] + "..."
|
||||
|
||||
|
||||
def _slim_for_llm(data):
|
||||
"""Return a trimmed copy of the context dict for the LLM prompt.
|
||||
|
||||
Keeps only the fields the model needs and truncates long bodies so the
|
||||
serialized JSON stays small enough for fast prefill.
|
||||
"""
|
||||
slim = dict(data)
|
||||
|
||||
if isinstance(data.get("releases"), list):
|
||||
slim_releases = []
|
||||
for rel in data["releases"]:
|
||||
if not isinstance(rel, dict):
|
||||
slim_releases.append(rel)
|
||||
continue
|
||||
slim_releases.append({
|
||||
"tag_name": rel.get("tag_name"),
|
||||
"name": rel.get("name"),
|
||||
"published_at": rel.get("published_at"),
|
||||
"body": _truncate(rel.get("body", ""), LLM_BODY_LIMIT),
|
||||
})
|
||||
slim["releases"] = slim_releases
|
||||
|
||||
if isinstance(data.get("issues"), list):
|
||||
slim_issues = []
|
||||
for issue in data["issues"]:
|
||||
if not isinstance(issue, dict):
|
||||
slim_issues.append(issue)
|
||||
continue
|
||||
item = {
|
||||
"title": issue.get("title"),
|
||||
"state": issue.get("state"),
|
||||
"created_at": issue.get("created_at"),
|
||||
}
|
||||
if "body" in issue:
|
||||
item["body"] = _truncate(issue.get("body") or "", LLM_FIELD_LIMIT)
|
||||
slim_issues.append(item)
|
||||
slim["issues"] = slim_issues
|
||||
|
||||
if isinstance(data.get("messages"), list):
|
||||
slim["messages"] = [
|
||||
_truncate(m, LLM_MESSAGE_LIMIT) if isinstance(m, str)
|
||||
else {**m, "content": _truncate(m.get("content", ""), LLM_MESSAGE_LIMIT)} if isinstance(m, dict)
|
||||
else m
|
||||
for m in data["messages"]
|
||||
]
|
||||
|
||||
serialized = json.dumps(slim, ensure_ascii=False, default=str)
|
||||
if len(serialized) > LLM_TOTAL_LIMIT:
|
||||
if isinstance(slim.get("releases"), list):
|
||||
slim["releases"] = slim["releases"][:3]
|
||||
if isinstance(slim.get("issues"), list):
|
||||
slim["issues"] = slim["issues"][:5]
|
||||
logger.info(
|
||||
"_slim_for_llm: payload still >%d chars, cut to %d releases / %d issues",
|
||||
LLM_TOTAL_LIMIT,
|
||||
len(slim.get("releases", [])),
|
||||
len(slim.get("issues", [])),
|
||||
)
|
||||
return slim
|
||||
|
||||
|
||||
def build_prompts(data, question):
|
||||
"""Assemble system/user messages + the human-readable prompt display."""
|
||||
slim_data = _slim_for_llm(data)
|
||||
data_json = json.dumps(slim_data, ensure_ascii=False, default=str)
|
||||
user_prompt = f"分析数据: {data_json}\n\n用户问题: {question}\n\n{FORMAT_REQ}"
|
||||
messages = [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
full_prompt_display = f"[SYSTEM]\n{SYSTEM_PROMPT}\n\n[USER]\n{user_prompt}"
|
||||
return messages, full_prompt_display
|
||||
|
||||
|
||||
def _sse(event, payload):
|
||||
return f"event: {event}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
||||
|
||||
|
||||
def stream_analysis_events(data, question):
|
||||
"""Generator yielding SSE frames: step/token/section_start/error/complete.
|
||||
|
||||
Pure of Flask: safe to consume after the request context has closed,
|
||||
as long as `data` and `question` were prepared inside the view.
|
||||
"""
|
||||
start_time = time.time()
|
||||
data_summary = data["data_summary"]
|
||||
messages, full_prompt_display = build_prompts(data, question)
|
||||
|
||||
yield _sse("step", {"step": "context", "content": f"已收集上下文: {data_summary}"})
|
||||
yield _sse("step", {"step": "prompt", "content": full_prompt_display})
|
||||
|
||||
def emit_token(t):
|
||||
return _sse("token", {"text": t})
|
||||
|
||||
try:
|
||||
resp = http_requests.post(
|
||||
f"{config.LLM_URL}/chat/completions",
|
||||
json={"model": config.LLM_MODEL, "messages": messages, "stream": True, "max_tokens": 1024},
|
||||
stream=True,
|
||||
timeout=config.LLM_TIMEOUT,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
last_keepalive = time.time()
|
||||
holdback = max(len(DELIM_REASONING), len(DELIM_CONCLUSION)) - 1
|
||||
pending = ""
|
||||
reasoning_started = False
|
||||
conclusion_started = False
|
||||
|
||||
for line in resp.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
line = line.decode("utf-8") if isinstance(line, bytes) else line
|
||||
|
||||
now = time.time()
|
||||
if now - last_keepalive > 15:
|
||||
yield ": keepalive\n\n"
|
||||
last_keepalive = now
|
||||
|
||||
if line.startswith("data: "):
|
||||
chunk_data = line[6:]
|
||||
if chunk_data.strip() == "[DONE]":
|
||||
break
|
||||
try:
|
||||
chunk = json.loads(chunk_data)
|
||||
delta = chunk.get("choices", [{}])[0].get("delta", {})
|
||||
token_text = delta.get("content", "")
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not token_text:
|
||||
continue
|
||||
|
||||
pending += token_text
|
||||
|
||||
# 滚动 buffer: 分隔符可能跨 chunk, 保留尾部前缀再下发
|
||||
while pending:
|
||||
idx_r = pending.find(DELIM_REASONING)
|
||||
idx_c = pending.find(DELIM_CONCLUSION)
|
||||
idx = -1
|
||||
delim = None
|
||||
if idx_r != -1 and (idx_c == -1 or idx_r < idx_c):
|
||||
idx, delim = idx_r, DELIM_REASONING
|
||||
elif idx_c != -1:
|
||||
idx, delim = idx_c, DELIM_CONCLUSION
|
||||
|
||||
if idx != -1:
|
||||
if idx > 0:
|
||||
if not reasoning_started:
|
||||
reasoning_started = True
|
||||
yield "event: reasoning_start\ndata: {}\n\n"
|
||||
yield emit_token(pending[:idx])
|
||||
pending = pending[idx + len(delim):]
|
||||
if delim == DELIM_REASONING:
|
||||
if not reasoning_started:
|
||||
reasoning_started = True
|
||||
yield "event: reasoning_start\ndata: {}\n\n"
|
||||
else:
|
||||
if not reasoning_started:
|
||||
reasoning_started = True
|
||||
yield "event: reasoning_start\ndata: {}\n\n"
|
||||
if not conclusion_started:
|
||||
conclusion_started = True
|
||||
yield "event: conclusion_start\ndata: {}\n\n"
|
||||
continue
|
||||
|
||||
keep = 0
|
||||
tail = pending[-holdback:] if len(pending) > holdback else pending
|
||||
for d in (DELIM_REASONING, DELIM_CONCLUSION):
|
||||
for k in range(min(len(tail), len(d) - 1), 0, -1):
|
||||
if tail.endswith(d[:k]):
|
||||
keep = max(keep, k)
|
||||
break
|
||||
safe = pending[:len(pending) - keep] if keep else pending
|
||||
pending = pending[len(safe):]
|
||||
if not safe:
|
||||
break
|
||||
if not reasoning_started:
|
||||
reasoning_started = True
|
||||
yield "event: reasoning_start\ndata: {}\n\n"
|
||||
yield emit_token(safe)
|
||||
|
||||
if pending:
|
||||
if not reasoning_started:
|
||||
reasoning_started = True
|
||||
yield "event: reasoning_start\ndata: {}\n\n"
|
||||
yield emit_token(pending)
|
||||
|
||||
except Exception as e:
|
||||
yield _sse("error", {"error": f"LLM call failed: {str(e)}"})
|
||||
return
|
||||
|
||||
# 防御: 模型没给分隔符时补发 section_start, 保证前端 tab 一定有入口
|
||||
if not reasoning_started:
|
||||
yield "event: reasoning_start\ndata: {}\n\n"
|
||||
if not conclusion_started:
|
||||
yield "event: conclusion_start\ndata: {}\n\n"
|
||||
|
||||
elapsed = round(time.time() - start_time, 1)
|
||||
yield _sse("step", {"step": "complete", "content": f"分析完成, 总耗时 {elapsed} 秒"})
|
||||
|
||||
|
||||
def summarize_with_llm(messages, model=None, timeout=None):
|
||||
"""Non-streaming LLM call returning the assistant content string.
|
||||
|
||||
Kept Flask-free so scripts/daily_watchdog.py (and future cron jobs) can
|
||||
reuse the same endpoint configuration.
|
||||
"""
|
||||
resp = http_requests.post(
|
||||
f"{config.LLM_URL}/chat/completions",
|
||||
json={
|
||||
"model": model or config.LLM_MODEL,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"max_tokens": 1024,
|
||||
},
|
||||
timeout=timeout or config.LLM_TIMEOUT,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
Reference in New Issue
Block a user