init the project

This commit is contained in:
evyzacq
2026-05-25 15:09:42 +08:00
commit 7fc0e7852e
122 changed files with 14557 additions and 0 deletions
View File
+46
View File
@@ -0,0 +1,46 @@
# IR 版本差异对比服务
import logging
import yaml
from server.core.utils.diff import compute_diff
from server.services.ir_engine.generator import ir_generator
logger = logging.getLogger(__name__)
class IRDiffService:
"""Compare two IR versions and return a structured diff."""
async def diff(self, ir_id_a: str, ir_id_b: str) -> dict:
"""Compute diff between two IR versions."""
ir_a = await ir_generator.get_ir(ir_id_a)
ir_b = await ir_generator.get_ir(ir_id_b)
if not ir_a or not ir_b:
return {"error": "One or both IR versions not found", "diff": {}}
yaml_a = ir_a.get("yaml_content", "")
yaml_b = ir_b.get("yaml_content", "")
try:
data_a = yaml.safe_load(yaml_a)
data_b = yaml.safe_load(yaml_b)
except yaml.YAMLError as e:
return {"error": f"YAML parsing error: {e}", "diff": {}}
diff_result = compute_diff(data_a or {}, data_b or {})
logger.info("IR diff: %d added, %d removed, %d modified",
len(diff_result.get("added", [])),
len(diff_result.get("removed", [])),
len(diff_result.get("modified", [])))
return {
"ir_id_a": ir_id_a,
"ir_id_b": ir_id_b,
"diff": diff_result,
}
ir_diff = IRDiffService()
+76
View File
@@ -0,0 +1,76 @@
# IR Generator — delegates to the 3-stage pipeline
import logging
from server.services.ir_engine.pipeline import ir_pipeline
from server.services.prd_manager.service import prd_service
logger = logging.getLogger(__name__)
class IRGenerator:
"""IR generator that runs the full 3-stage pipeline on parsed PRD documents."""
async def generate(self, prd_id: str, prd_text: str = "", skill_name: str = "default") -> dict:
"""Run the pipeline on a parsed PRD document."""
# Get the parsed document from the PRD service
prd = await prd_service.get_prd(prd_id)
if not prd:
return {"error": f"PRD not found: {prd_id}", "ir_id": ""}
# Fetch full parsed data from disk if available
parsed_doc = self._load_parsed(prd)
if not parsed_doc:
# Fallback: build minimal doc from PRD text
parsed_doc = {
"source": "",
"sections": [{"source": "正文", "blocks": [{"type": "para", "index": 1, "text": prd.get("full_text", "")}], "images": []}],
"image_sources": {},
"image_analysis": [],
"resolved_conflicts": [],
}
result = await ir_pipeline.run(prd_id, parsed_doc)
return {
"ir_id": result["ir_id"],
"prd_id": result["prd_id"],
"yaml_content": result["yaml_content"],
"ir_json": result["ir_json"],
"audit": result["audit"],
"audit_report": result["audit_report"],
"skill_used": skill_name,
"created_at": result["created_at"],
"pipeline_stats": result["pipeline_stats"],
}
async def get_ir(self, ir_id: str) -> dict | None:
return await ir_pipeline.get_ir(ir_id)
def _load_parsed(self, prd: dict) -> dict | None:
"""Load the full parsed JSON from disk if available."""
import json
parsed_path = prd.get("parsed_path", "")
if parsed_path:
try:
with open(parsed_path, "r", encoding="utf-8") as f:
return json.load(f)
except Exception:
pass
# Build from in-memory data
if prd.get("sections") or prd.get("full_text"):
sections = prd.get("sections", [])
if not sections and prd.get("full_text"):
sections = [{"source": "正文", "blocks": [{"type": "para", "index": 1, "text": prd["full_text"]}], "images": []}]
return {
"source": "",
"sections": sections,
"image_sources": {},
"image_analysis": prd.get("images", []),
"resolved_conflicts": [],
}
return None
ir_generator = IRGenerator()
+753
View File
@@ -0,0 +1,753 @@
# 3-Stage IR Generation Pipeline
#
# Stage 1: Semantic Index — full document → function_units + concepts
# Stage 2: Per-Unit IR Extraction — precision context → detailed IR rules
# Stage 3: Deterministic Merge & Audit — dedup, assign IDs, coverage report
import concurrent.futures
import hashlib
import json
import logging
import re
import time
import uuid
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
from server.config import settings
from server.core.llm_provider.router import router
logger = logging.getLogger(__name__)
PROMPTS_DIR = Path(__file__).parent.parent / "prompts" # services/prompts/
MAX_CONCURRENCY = 5
PASS, WARN, FAIL = "[PASS]", "[WARN]", "[FAIL]"
# =============================================================================
# Stage 1: Semantic Index Generation
# =============================================================================
def format_document_for_prompt(doc: dict) -> str:
"""Render the full parsed document as a readable string for the LLM prompt."""
lines = ["=== SECTIONS ==="]
for i, section in enumerate(doc.get("sections", [])):
source = section.get("source", f"(无标题-章节{i})")
lines.append(f"\n--- Section: {source} ---")
for block in section.get("blocks", []):
if block["type"] == "para":
lines.append(f"[段落 {block['index']}] {block['text']}")
elif block["type"] == "table":
lines.append(f"[表格 {block.get('table', '?')}]")
headers = block.get("headers", [])
lines.append(f" 表头: {' | '.join(headers)}")
for row in block.get("rows", []):
cols = row.get("columns", [])
cell_texts = []
for c in cols:
cell_texts.append(f"[行{c.get('row','?')}]{c.get('name','')}: {c.get('text','')}")
lines.append(f" {'; '.join(cell_texts)}")
images = section.get("images", [])
if images:
lines.append(f" 图片引用: {', '.join(images)}")
# Image Analysis
lines.append("\n\n=== IMAGE_ANALYSIS ===")
for img in doc.get("image_analysis", []):
rid = img.get("rid", "?")
img_type = img.get("type", "?")
lines.append(f"\n--- Image: {rid} (type={img_type}) ---")
lines.append(f" 描述: {img.get('description', '')[:500]}")
# Resolved Conflicts
conflicts = doc.get("resolved_conflicts", [])
if conflicts:
lines.append("\n\n=== RESOLVED_CONFLICTS ===")
for c in conflicts:
lines.append(
f" [{c.get('conflict_type','?')}] {c.get('section','?')}: "
f"{c.get('source','?')}为准 — {c.get('correction','')}"
)
return "\n".join(lines)
def extract_json_from_response(text: str, expect_array: bool = False) -> str:
"""Robustly extract JSON object or array from LLM response."""
open_char = "[" if expect_array else "{"
close_char = "]" if expect_array else "}"
# Try code fence first
pattern = rf"```(?:json)?\s*(\[{open_char}[\s\S]*?\{close_char}\])\s*```" if expect_array else rf"```(?:json)?\s*(\{{[\s\S]*?\}})\s*```"
m = re.search(pattern, text)
if m:
return m.group(1).strip()
# Find outermost bracket
start = text.find(open_char)
if start == -1:
raise ValueError(f"No JSON {'array' if expect_array else 'object'} found in LLM response")
depth = 0
for i in range(start, len(text)):
if text[i] == open_char:
depth += 1
elif text[i] == close_char:
depth -= 1
if depth == 0:
return text[start: i + 1]
raise ValueError(f"Unclosed JSON {'array' if expect_array else 'object'} in LLM response")
async def stage1_semantic_index(doc: dict) -> dict:
"""Stage 1: Generate semantic index from full document."""
logger.info("[PIPELINE] ═══ Stage 1/3: 语义索引 ═══")
template_path = PROMPTS_DIR / "ir_step1_semantic_index.txt"
if not template_path.exists():
logger.warning("[PIPELINE] Stage 1 prompt template not found, using fallback")
return _fallback_semantic_index(doc)
template = template_path.read_text(encoding="utf-8")
formatted_doc = format_document_for_prompt(doc)
prompt = template.replace("{document_json}", formatted_doc)
logger.info("[PIPELINE] Prompt: %d chars (~%d tokens)", len(prompt), len(prompt) // 3)
client = router.get_text_client()
for attempt in range(3):
try:
raw = client.chat(
model=router.text_model,
messages=[
{"role": "system", "content": "你是一个精确的 JSON 输出引擎。只输出合法的 JSON,不输出任何其他文字。"},
{"role": "user", "content": prompt},
],
temperature=0.1,
)
json_str = extract_json_from_response(raw)
result = json.loads(json_str)
# Validate
if "function_units" not in result or not result["function_units"]:
raise ValueError("Missing function_units")
logger.info("[PIPELINE] ✅ Stage 1 完成: %d function_units, %d concepts",
len(result["function_units"]), len(result.get("concepts", [])))
return result
except Exception as e:
logger.warning("[PIPELINE] Stage 1 attempt %d failed: %s", attempt + 1, e)
if attempt < 2:
time.sleep(2)
# Fallback
logger.warning("[PIPELINE] Stage 1 全部重试失败 exhausted, using fallback")
return _fallback_semantic_index(doc)
def _fallback_semantic_index(doc: dict) -> dict:
"""Generate a basic semantic index without LLM (for mock/demo)."""
units = []
unit_idx = 1
for section in doc.get("sections", []):
source = section.get("source", "")
for block in section.get("blocks", []):
if block["type"] == "para":
# Create a function unit from each substantial paragraph
text = block.get("text", "")
if len(text) > 30:
units.append({
"unit_id": f"FU-{unit_idx:03d}",
"name": text[:60],
"description": text[:200],
"sources": [{"section": source, "type": "para", "text_snippet": text[:200]}],
})
unit_idx += 1
elif block["type"] == "table":
for row in block.get("rows", []):
cols = row.get("columns", [])
texts = [f"{c.get('name','')}: {c.get('text','')}" for c in cols if c.get('text')]
if texts:
combined = "; ".join(texts)
units.append({
"unit_id": f"FU-{unit_idx:03d}",
"name": combined[:60],
"description": combined[:200],
"sources": [{"section": source, "type": "table", "row": row.get("columns", [{}])[0].get("row", 1) if cols else 1, "text_snippet": combined[:200]}],
})
unit_idx += 1
return {
"feature_name": doc.get("sections", [{}])[0].get("source", "未命名功能"),
"concepts": [],
"function_units": units,
}
# =============================================================================
# Stage 2: Per-Unit IR Extraction
# =============================================================================
def build_context_package(fu: dict, doc: dict, image_sources: dict) -> dict:
"""Build a precision context package for one function unit."""
texts = []
tables = []
images = []
seen_sections = set()
img_by_rid = {img.get("rid", ""): img for img in doc.get("image_analysis", [])}
for src in fu.get("sources", []):
section_key = src.get("section", "")
src_type = src.get("type", "")
if src_type in ("para", "table") and section_key:
if section_key in seen_sections:
continue
seen_sections.add(section_key)
# Find matching section
for section in doc.get("sections", []):
if section.get("source", "") == section_key:
for block in section.get("blocks", []):
if block["type"] == "para":
texts.append({
"section": section_key,
"text": block["text"],
})
elif block["type"] == "table":
all_rows = []
for row in block.get("rows", []):
for col in row.get("columns", []):
all_rows.append({
"row": col.get("row"),
"name": col.get("name"),
"text": col.get("text"),
})
# Trim: max 20 cells per table to reduce prompt size
tables.append({
"section": section_key,
"headers": block.get("headers", []),
"all_rows": all_rows[:20],
"_truncated": len(all_rows) > 20,
})
break
if src_type in ("image", "logic_tree"):
img_id = src.get("image_id", "")
if img_id and img_id in img_by_rid:
img = img_by_rid[img_id]
images.append({
"image_id": img_id,
"type": img.get("type", "other"),
"description": img.get("description", ""),
})
return {
"unit_id": fu["unit_id"],
"unit_name": fu.get("name", ""),
"unit_description": fu.get("description", ""),
"texts": texts,
"tables": tables,
"images": images,
"resolved_conflicts": doc.get("resolved_conflicts", []),
}
async def stage2_extract_rules(semantic_index: dict, doc: dict) -> list[dict]:
"""Stage 2: Extract detailed IR rules for each function unit."""
logger.info("=" * 50)
logger.info("[PIPELINE] ═══ Stage 2/3: Per-Unit IR Extraction")
template_path = PROMPTS_DIR / "ir_step2_extraction.txt"
if not template_path.exists():
return _fallback_rules(semantic_index)
template = template_path.read_text(encoding="utf-8")
function_units = semantic_index.get("function_units", [])
image_sources = doc.get("image_sources", {})
logger.info("Processing %d function units (max concurrency: %d)", len(function_units), MAX_CONCURRENCY)
# Build context packages
packages = []
for fu in function_units:
pkg = build_context_package(fu, doc, image_sources)
packages.append(pkg)
# Parallel LLM calls
fragments = []
client = router.get_text_client()
def extract_one(pkg: dict) -> dict:
"""Extract rules for one function unit."""
prompt = _build_stage2_prompt(template, pkg)
unit_id = pkg["unit_id"]
for attempt in range(3):
try:
raw = client.chat(
model=router.text_model,
messages=[
{"role": "system", "content": "你是一个精确的 JSON 输出引擎。只输出合法的 JSON 数组,不输出任何其他文字。"},
{"role": "user", "content": prompt},
],
temperature=0.1,
)
json_str = extract_json_from_response(raw, expect_array=True)
rules = json.loads(json_str)
if not isinstance(rules, list):
raise ValueError(f"Expected array, got {type(rules).__name__}")
return {"unit_id": unit_id, "unit_name": pkg["unit_name"], "rules": rules}
except Exception as e:
if attempt < 2:
time.sleep(2)
else:
logger.warning("Unit %s failed after 3 attempts: %s", unit_id, e)
return {"unit_id": unit_id, "unit_name": pkg["unit_name"], "rules": [], "error": str(e)}
return {"unit_id": unit_id, "unit_name": pkg["unit_name"], "rules": [], "error": "unknown"}
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENCY) as executor:
futures = {executor.submit(extract_one, pkg): pkg for pkg in packages}
for future in concurrent.futures.as_completed(futures):
result = future.result()
fragments.append(result)
n_rules = len(result.get("rules", []))
logger.info("[PIPELINE] %s: %d rules %s", result["unit_id"], n_rules,
"" if result.get("error") else "")
fragments.sort(key=lambda f: f["unit_id"])
total = sum(len(f.get("rules", [])) for f in fragments)
logger.info("[PIPELINE] ✅ Stage 2 完成: %d fragments, %d total rules", len(fragments), total)
return fragments
def _build_stage2_prompt(template: str, pkg: dict) -> str:
"""Render the stage 2 prompt template with context package data."""
# Simple str.replace (avoids format issues with curly braces in JSON)
prompt = template
prompt = prompt.replace("{unit_id}", pkg["unit_id"])
prompt = prompt.replace("{unit_name}", pkg["unit_name"])
prompt = prompt.replace("{unit_description}", pkg["unit_description"])
prompt = prompt.replace("{texts}", json.dumps(pkg.get("texts", []), ensure_ascii=False, indent=2))
prompt = prompt.replace("{tables}", json.dumps(pkg.get("tables", []), ensure_ascii=False, indent=2))
prompt = prompt.replace("{images}", json.dumps(pkg.get("images", []), ensure_ascii=False, indent=2))
prompt = prompt.replace("{resolved_conflicts}", json.dumps(pkg.get("resolved_conflicts", []), ensure_ascii=False, indent=2))
return prompt
def _fallback_rules(semantic_index: dict) -> list[dict]:
"""Generate basic rules without LLM."""
fragments = []
for fu in semantic_index.get("function_units", []):
fragments.append({
"unit_id": fu["unit_id"],
"unit_name": fu.get("name", ""),
"rules": [{
"description": fu.get("description", ""),
"priority": "P2",
"sources": fu.get("sources", []),
"precondition": {},
"trigger": {"operator": "AND", "conditions": []},
"actions": [],
}],
})
return fragments
# =============================================================================
# Stage 3: Merge & Audit
# =============================================================================
def rule_signature(rule: dict) -> str:
"""SHA256 signature from trigger + actions for dedup."""
trigger = rule.get("trigger", {})
actions = rule.get("actions", [])
conditions = sorted(trigger.get("conditions", []), key=lambda c: json.dumps(c, sort_keys=True))
sorted_actions = sorted(actions, key=lambda a: json.dumps(a, sort_keys=True))
sig = json.dumps({"conditions": conditions, "actions": sorted_actions}, ensure_ascii=False, sort_keys=True)
return hashlib.sha256(sig.encode()).hexdigest()[:16]
def stage3_merge_and_audit(fragments: list[dict], semantic_index: dict, doc: dict) -> dict:
"""Stage 3: Merge rules, assign IDs, and generate audit report."""
logger.info("=" * 50)
logger.info("[PIPELINE] ═══ Stage 3/3: Merge & Audit")
# Merge and dedup
sig_map: dict[str, dict] = {}
order = []
for f in fragments:
for rule in f.get("rules", []):
sig = rule_signature(rule)
if sig in sig_map:
existing = sig_map[sig]
existing_sources = existing.setdefault("sources", [])
for src in rule.get("sources", []):
if src not in existing_sources:
existing_sources.append(src)
if len(rule.get("description", "")) > len(existing.get("description", "")):
existing["description"] = rule["description"]
else:
sig_map[sig] = dict(rule)
order.append(sig)
merged = [sig_map[sig] for sig in order]
logger.info("[PIPELINE] 合并: %d%d rules (dedup)", sum(len(f.get("rules", [])) for f in fragments), len(merged))
# Assign rule IDs: IR-{feature_id}-{section}-{seq}
section_counters = defaultdict(int)
feature_name = semantic_index.get("feature_name", "FEATURE")
feature_id = re.sub(r'[^A-Z]', '', feature_name.upper())[:6] or "FEAT"
if len(feature_id) < 2:
feature_id = (feature_id + "FEATURE")[:4]
for rule in merged:
# Extract section from first para/table source, fallback to "0"
section = "0"
for src in rule.get("sources", []):
s = src.get("section", "")
if s:
# Extract first number pattern like "4.2.1" or "3.1"
m = re.search(r'[\d]+(?:\.[\d]+)*', s)
if m:
section = m.group()
break
section_counters[section] += 1
rule["rule_id"] = f"IR-{feature_id}-{section}-{section_counters[section]:03d}"
# Build final IR
ir_final = {
"feature": feature_name,
"feature_id": feature_id,
"rules": merged,
"rule_count": len(merged),
"generated_at": datetime.now(timezone.utc).isoformat(),
}
# Generate audit report
audit_results = _generate_audit(merged, doc)
report = _format_audit_report(feature_name, len(merged), audit_results)
logger.info("[PIPELINE] ✅ Stage 3 完成: %d rules, %d audit issues", len(merged), audit_results.get("issue_count", 0))
return {"ir": ir_final, "audit": audit_results, "audit_report": report}
def _generate_audit(rules: list[dict], doc: dict) -> dict:
"""Run coverage audits and return structured results."""
results = []
# 1. Source coverage: check that all sections with content are referenced
all_sources = set()
for rule in rules:
for src in rule.get("sources", []):
section = src.get("section", "")
if section:
all_sources.add(section)
doc_sections = set()
for section in doc.get("sections", []):
source = section.get("source", "")
if source and section.get("blocks"):
doc_sections.add(source)
uncovered = doc_sections - all_sources
results.append({
"check": "文档章节覆盖",
"status": PASS if not uncovered else WARN,
"detail": f"已覆盖 {len(all_sources)}/{len(doc_sections)} 章节"
+ (f"; 未覆盖: {list(uncovered)[:5]}" if uncovered else ""),
})
# 2. Image coverage
img_count = len(doc.get("image_analysis", []))
img_referenced = set()
for rule in rules:
for src in rule.get("sources", []):
if src.get("type") in ("image", "logic_tree"):
img_referenced.add(src.get("image_id", ""))
results.append({
"check": "图片覆盖率",
"status": PASS if len(img_referenced) >= img_count else (WARN if img_referenced else FAIL),
"detail": f"已引用 {len(img_referenced)}/{img_count} 张图片",
})
# 3. Priority distribution
p0 = sum(1 for r in rules if r.get("priority") == "P0")
p1 = sum(1 for r in rules if r.get("priority") == "P1")
p2 = sum(1 for r in rules if r.get("priority") == "P2")
results.append({
"check": "优先级分布",
"status": PASS if p0 > 0 else WARN,
"detail": f"P0={p0}, P1={p1}, P2={p2}",
})
issue_count = sum(1 for r in results if r["status"] in (WARN, FAIL))
return {"checks": results, "issue_count": issue_count}
def _sync_call(async_fn, *args, **kwargs):
"""Run an async function synchronously using the existing event loop if available."""
import asyncio
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# No event loop running — just use asyncio.run
return asyncio.run(async_fn(*args, **kwargs))
else:
# Event loop is running (e.g., inside FastAPI) — must use run_until_complete
# Create a new loop in a thread to avoid conflicts
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
future = ex.submit(asyncio.run, async_fn(*args, **kwargs))
return future.result()
def _format_audit_report(feature_name: str, rule_count: int, audit: dict) -> str:
"""Generate Markdown audit report."""
lines = [
f"# IR 完整性审计报告",
f"",
f"**功能**: {feature_name}",
f"**规则总数**: {rule_count}",
f"**生成时间**: {datetime.now(timezone.utc).isoformat()}",
f"",
f"> ⚠️ 请人工审查以下 ⚠️ 和 ❌ 项。",
f"",
f"## 审计结果",
f"",
f"| 检查项 | 状态 | 详情 |",
f"|--------|------|------|",
]
for r in audit.get("checks", []):
lines.append(f"| {r['check']} | {r['status']} | {r['detail']} |")
lines.append("")
lines.append(f"**审计问题总计**: {audit.get('issue_count', 0)} 个需要人工审查")
return "\n".join(lines)
# =============================================================================
# Pipeline Orchestrator
# =============================================================================
class IRPipeline:
"""Orchestrates the 3-stage IR generation pipeline."""
def __init__(self):
self._store: dict[str, dict] = {}
def run_streaming(self, prd_id: str, parsed_doc: dict):
"""Run pipeline as a sync generator yielding progress events.
This is a plain generator (NOT async) so it never blocks the event loop
between yields. The SSE endpoint wraps it in a thread + Queue bridge.
"""
t_total_start = time.time()
n_sections = len(parsed_doc.get("sections", []))
est_units = max(n_sections * 2, 3)
est_total = 20 + (est_units * 20 / 3) + 1
# --- Stage 1 ---
yield {"stage": 1, "status": "running", "message": "Stage 1: 分析文档结构,识别功能单元...",
"elapsed": 0, "stage_total": 3, "detail": f"文档含 {n_sections} 个章节"}
t0 = time.time()
try:
semantic_index = _sync_call(stage1_semantic_index, parsed_doc)
except Exception as e:
yield {"stage": 1, "status": "error", "message": f"Stage 1 失败: {e}", "stage_total": 3}
return
t1 = time.time()
n_units = len(semantic_index.get("function_units", []))
est_total = 20 + (max(n_units, 1) * 25 / 3) + 1
yield {"stage": 1, "status": "done", "message": f"Stage 1 完成: 识别 {n_units} 个功能单元",
"elapsed": round(t1 - t0, 1), "stage_total": 3, "detail": f"识别 {n_units} 个功能单元",
"n_units": n_units, "estimated_total": round(est_total)}
# --- Stage 2 ---
est_stage2 = max(n_units, 1) * 25 / min(MAX_CONCURRENCY, max(n_units, 1))
yield {"stage": 2, "status": "running", "message": f"Stage 2: 逐单元提取 IR 规则 ({n_units} 个单元)...",
"elapsed": 0, "stage_total": 3, "detail": f"预计 {est_stage2:.0f}s{n_units} 个单元 × {min(MAX_CONCURRENCY, n_units)} 并发",
"n_units": n_units, "estimated_total": round(est_total)}
t1b = time.time()
completed = 0
total_rules = 0
fragments = []
function_units = semantic_index.get("function_units", [])
# Optimization: filter out trivial units (very short desc, no text sources)
filtered_units = []
for fu in function_units:
sources = fu.get("sources", [])
desc = fu.get("description", "")
has_text = any(s.get("type") in ("para", "table") for s in sources)
if len(desc) < 15 and not has_text:
logger.info("[PIPELINE] 跳过空单元: %s", fu.get("unit_id", "?"))
continue
filtered_units.append(fu)
skipped = len(function_units) - len(filtered_units)
if skipped:
logger.info("[PIPELINE] Stage 2 过滤掉 %d 个空单元,剩余 %d", skipped, len(filtered_units))
function_units = filtered_units
image_sources = parsed_doc.get("image_sources", {})
template_path = PROMPTS_DIR / "ir_step2_extraction.txt"
template = template_path.read_text(encoding="utf-8") if template_path.exists() else ""
client = router.get_text_client()
packages = [build_context_package(fu, parsed_doc, image_sources) for fu in function_units]
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENCY) as executor:
futures = {}
for i, pkg in enumerate(packages):
future = executor.submit(self._extract_one_sync, template, pkg, client)
futures[future] = (i, pkg["unit_id"], pkg["unit_name"])
for future in concurrent.futures.as_completed(futures):
i, uid, uname = futures[future]
try:
rules = future.result()
fragments.append({"unit_id": uid, "unit_name": uname, "rules": rules})
total_rules += len(rules)
except Exception as e:
logger.warning("Unit %s failed: %s", uid, e)
fragments.append({"unit_id": uid, "unit_name": uname, "rules": [], "error": str(e)})
completed += 1
elapsed = round(time.time() - t1b, 1)
yield {"stage": 2, "status": "running",
"message": f"Stage 2: {completed}/{n_units} 单元完成 ({total_rules} 条规则)",
"elapsed": elapsed, "stage_total": 3,
"detail": f"已完成 {completed}/{n_units},累计 {total_rules} 条规则",
"completed": completed, "total": n_units, "rules_so_far": total_rules,
"estimated_total": round(est_total)}
fragments.sort(key=lambda f: f["unit_id"])
t2 = time.time()
logger.info("[PIPELINE] ✅ Stage 2 完成: %d fragments, %d total rules", len(fragments), total_rules)
yield {"stage": 2, "status": "done",
"message": f"Stage 2 完成: {len(fragments)} 个单元 → {total_rules} 条规则",
"elapsed": round(t2 - t1b, 1), "stage_total": 3,
"detail": f"{len(fragments)} 个单元 → {total_rules} 条规则",
"total_rules": total_rules}
# --- Stage 3 ---
yield {"stage": 3, "status": "running", "message": "Stage 3: 合并去重 + 审计报告...",
"elapsed": 0, "stage_total": 3, "detail": "去重 + 分配 ID + 生成审计报告"}
result = stage3_merge_and_audit(fragments, semantic_index, parsed_doc)
t3 = time.time()
n_rules = len(result["ir"]["rules"])
ir_id = str(uuid.uuid4())[:8]
output = {
"ir_id": ir_id, "prd_id": prd_id,
"yaml_content": json.dumps(result["ir"], ensure_ascii=False, indent=2),
"ir_json": result["ir"],
"audit": result["audit"],
"audit_report": result["audit_report"],
"skill_used": "default",
"created_at": datetime.now(timezone.utc).isoformat(),
"pipeline_stats": {
"stage1_seconds": round(t1 - t0, 1),
"stage2_seconds": round(t2 - t1b, 1),
"stage3_seconds": round(t3 - t2, 1),
"total_seconds": round(t3 - t_total_start, 1),
},
}
self._store[ir_id] = output
# Save to disk
output_dir = settings.OUTPUT_DIR / prd_id
output_dir.mkdir(parents=True, exist_ok=True)
with open(output_dir / f"ir_{ir_id}.json", "w", encoding="utf-8") as f:
json.dump(output, f, ensure_ascii=False, indent=2, default=str)
yield {"stage": 3, "status": "done", "message": f"流水线完成!共 {n_rules} 条规则",
"elapsed": round(t3 - t2, 1), "stage_total": 3,
"detail": f"合并后 {n_rules} 条规则,审计问题 {result['audit'].get('issue_count', 0)}",
"done": True, "result": output}
def _extract_one_sync(self, template: str, pkg: dict, client) -> list[dict]:
"""Synchronous rule extraction for one unit (runs in thread pool)."""
prompt = _build_stage2_prompt(template, pkg)
for attempt in range(3):
try:
raw = client.chat(
model=router.text_model,
messages=[
{"role": "system", "content": "你是一个精确的 JSON 输出引擎。只输出合法的 JSON 数组,不输出任何其他文字。"},
{"role": "user", "content": prompt},
],
temperature=0.1,
)
json_str = extract_json_from_response(raw, expect_array=True)
rules = json.loads(json_str)
if isinstance(rules, list):
return rules
except Exception as e:
if attempt == 2:
raise
time.sleep(2)
return []
async def run(self, prd_id: str, parsed_doc: dict) -> dict:
"""Run the full 3-stage pipeline and return the final IR + audit."""
logger.info("[PIPELINE] ═══ 流水线启动 for PRD %s", prd_id)
# Stage 1
t0 = time.time()
semantic_index = await stage1_semantic_index(parsed_doc)
t1 = time.time()
logger.info("[PIPELINE] Stage 1 耗时: %.1fs", t1 - t0)
# Stage 2
fragments = await stage2_extract_rules(semantic_index, parsed_doc)
t2 = time.time()
logger.info("[PIPELINE] Stage 2 耗时 %.1fs", t2 - t1)
# Stage 3
result = stage3_merge_and_audit(fragments, semantic_index, parsed_doc)
t3 = time.time()
logger.info("[PIPELINE] Stage 3 耗时 %.1fs", t3 - t2)
ir_id = str(uuid.uuid4())[:8]
output = {
"ir_id": ir_id,
"prd_id": prd_id,
"yaml_content": json.dumps(result["ir"], ensure_ascii=False, indent=2),
"ir_json": result["ir"],
"audit": result["audit"],
"audit_report": result["audit_report"],
"skill_used": "default",
"created_at": datetime.now(timezone.utc).isoformat(),
"pipeline_stats": {
"stage1_seconds": round(t1 - t0, 1),
"stage2_seconds": round(t2 - t1, 1),
"stage3_seconds": round(t3 - t2, 1),
"total_seconds": round(t3 - t0, 1),
},
}
self._store[ir_id] = output
# Save output
output_dir = settings.OUTPUT_DIR / prd_id
output_dir.mkdir(parents=True, exist_ok=True)
with open(output_dir / f"ir_{ir_id}.json", "w", encoding="utf-8") as f:
json.dump(output, f, ensure_ascii=False, indent=2, default=str)
logger.info("[PIPELINE] ✅ 流水线完成: ir_id=%s, rules=%d, total=%.1fs",
ir_id, len(result["ir"]["rules"]), t3 - t0)
return output
async def get_ir(self, ir_id: str) -> dict | None:
return self._store.get(ir_id)
ir_pipeline = IRPipeline()
+100
View File
@@ -0,0 +1,100 @@
# IR 验证器:Schema 校验 + Principles 规则检查
import json
import logging
import jsonschema
import yaml
from server.config import settings
from server.services.ir_engine.generator import ir_generator
logger = logging.getLogger(__name__)
class IRValidator:
"""Validate IR against skill schema and soul principles."""
async def validate(self, ir_yaml: str, skill_name: str = "default") -> dict:
"""Validate IR YAML content and return {valid, issues}."""
issues = []
# 1. Parse YAML
try:
ir_data = yaml.safe_load(ir_yaml)
except yaml.YAMLError as e:
return {"valid": False, "issues": [{
"severity": "error",
"message": f"YAML 解析失败: {e}",
}]}
if not isinstance(ir_data, dict):
return {"valid": False, "issues": [{
"severity": "error",
"message": "IR 内容必须是 YAML 字典/对象",
}]}
# 2. Schema validation
skill = ir_generator.load_skill(skill_name)
schema = skill.get("schema", {})
if schema:
try:
jsonschema.validate(instance=ir_data, schema=schema)
except jsonschema.ValidationError as e:
issues.append({
"severity": "error",
"message": f"Schema 校验失败: {e.message}",
"location": ".".join(str(p) for p in e.absolute_path),
})
# 3. Principles validation
principles = self._load_principles()
for principle in principles:
rule = principle.get("rule", "")
pid = principle.get("id", "")
# Check completeness: each feature must have both positive and negative paths
if pid == "completeness":
features = ir_data.get("features", [])
if not features:
issues.append({
"severity": "warning",
"message": f"原则 '{pid}': 未检测到功能点,无法验证完整性",
})
# Check traceability: features should have descriptions
if pid == "traceability":
features = ir_data.get("features", [])
for i, feat in enumerate(features):
if not feat.get("description"):
issues.append({
"severity": "warning",
"message": f"原则 '{pid}': 功能点 {feat.get('feature_name', f'#{i}')} 缺少描述",
})
# Check no-hallucination: warn about potentially vague features
if pid == "no-hallucination":
features = ir_data.get("features", [])
for feat in features:
desc = feat.get("description", "")
if "应该" in desc or "可能" in desc or "猜测" in desc:
issues.append({
"severity": "warning",
"message": f"原则 '{pid}': 功能点 '{feat.get('feature_name', '')}' 描述包含不确定措辞",
})
valid = len([i for i in issues if i["severity"] == "error"]) == 0
logger.info("IR validation: valid=%s, issues=%d", valid, len(issues))
return {"valid": valid, "issues": issues}
def _load_principles(self) -> list[dict]:
"""Load principles from .zeekerwatchmen/soul/principles.yaml."""
principles_path = settings.SOUL_DIR / "principles.yaml"
if not principles_path.exists():
return []
with open(principles_path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
return data.get("principles", []) if data else []
ir_validator = IRValidator()
@@ -0,0 +1,85 @@
# Vision LLM wrapper for analyzing document images (type + description).
import logging
from typing import Optional
from server.core.llm_provider.router import router
logger = logging.getLogger(__name__)
PROMPT_IMAGE = """请分析这张图片,判断类型并输出文字描述。
## 判断图片类型
如果是 **流程图 / 架构图 / 状态图 / 时序图 / 活动图**,详细描述:
- 图中所有节点/步骤/状态/组件的名称
- 所有连线/箭头/转换关系及其方向
- 所有分支条件、判断逻辑和判断结果
- 所有文字标注、注释、标签
- 图的整体结构和逻辑流程
- 如果图片包含多个子图,拆解描述
如果是 **其他类型**(UI原型图 / 界面截图 / 设计稿 / 手机屏幕截图 / 网页截图等),简要描述图片内容。
## 输出格式
**1. 类型标签(单独一行):**
type: <flowchart|architecture|state|sequence|activity|other>
**2. 文字描述:**
该图片的详细文字描述。
不要输出 ---YAML--- 分隔符或 YAML 内容,不要添加任何额外的解释或问候语。"""
class ImageParser:
"""Analyze document images using Qwen VL model.
Usage::
parser = ImageParser()
result = parser.parse_image("images/flow.png")
# {"type": "flowchart", "description": "..."}
"""
_VALID_TYPES = {"flowchart", "architecture", "state", "sequence", "activity", "other"}
def __init__(self):
self._llm = router.get_image_client()
@property
def usage(self) -> dict:
return self._llm.usage
def parse_image(self, image_path: str) -> Optional[dict]:
"""Parse an image and return {type, description}."""
logger.info("Parsing image: %s", image_path)
try:
content = self._llm.chat_with_image(
model=router.image_model,
image_path=image_path,
prompt=PROMPT_IMAGE,
)
except Exception as e:
logger.error("Image parsing failed: %s", e)
return {"type": "other", "description": "", "error": str(e)}
return self._parse_response(content)
def _parse_response(self, content: str) -> dict:
"""Extract (type, description) from vision model response."""
content = content.strip()
parsed_type = "other"
desc_lines: list[str] = []
for line in content.splitlines():
stripped = line.strip()
if (stripped.startswith("type:") or stripped.startswith("类型:")):
type_val = stripped.split(":", 1)[1].strip().lower()
if type_val in self._VALID_TYPES:
parsed_type = type_val
else:
desc_lines.append(line)
return {"type": parsed_type, "description": "\n".join(desc_lines).strip()}
+260
View File
@@ -0,0 +1,260 @@
# PRD 管理服务:上传、解析、版本快照
import json
import logging
import os
import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
from server.config import settings
from server.services.prd_manager.word_parser import WordParser
from server.services.prd_manager.image_parser import ImageParser
logger = logging.getLogger("testflow")
RATE_LIMIT_DELAY = 0.5
class PRDService:
"""PRD management: handle file upload, text extraction, and version snapshots."""
def __init__(self):
self.output_dir = settings.OUTPUT_DIR
os.makedirs(self.output_dir, exist_ok=True)
self._prds: dict[str, dict] = {}
async def upload_and_parse(self, filename: str, content: bytes) -> dict:
prd_id = str(uuid.uuid4())[:8]
basename = Path(filename).stem
ext = Path(filename).suffix.lower()
upload_dir = self.output_dir / prd_id
os.makedirs(upload_dir, exist_ok=True)
filepath = upload_dir / filename
t0 = time.time()
with open(filepath, "wb") as f:
f.write(content)
logger.info("[UPLOAD] 收到文件: %s (%.1f KB, %s)", filename, len(content) / 1024, ext)
if ext in (".txt", ".md"):
text = content.decode("utf-8", errors="replace")
sections = self._parse_markdown_sections(text)
parsed = {
"source": str(filepath), "sections": sections,
"image_sources": {}, "image_analysis": [], "full_text": text,
}
logger.info("[PARSE] %.4s %s%d 章节, %d 字符, %.1fs",
prd_id, ext, len(sections), len(text), time.time() - t0)
elif ext == ".docx":
logger.info("[PARSE] %.4s → 开始解析 .docx...", prd_id)
parsed = await self._parse_docx(str(filepath), upload_dir)
logger.info("[PARSE] %.4s .docx 完成: %d 章节, %.1fs",
prd_id, len(parsed.get("sections", [])), time.time() - t0)
elif ext == ".pdf":
logger.info("[PARSE] %.4s → 开始解析 .pdf...", prd_id)
parsed = await self._parse_pdf(str(filepath), upload_dir)
logger.info("[PARSE] %.4s .pdf 完成: %d 页, %.1fs",
prd_id, len(parsed.get("sections", [])), time.time() - t0)
else:
raise ValueError(f"Unsupported file format: {ext}")
parsed_path = upload_dir / f"{basename}_parsed.json"
with open(parsed_path, "w", encoding="utf-8") as f:
json.dump(parsed, f, ensure_ascii=False, indent=2)
prd_version = {
"prd_id": prd_id, "filename": filename,
"uploaded_at": datetime.now(timezone.utc).isoformat(), "status": "ready",
"parsed_path": str(parsed_path), "full_text": parsed.get("full_text", ""),
"sections": parsed.get("sections", []), "images": parsed.get("image_analysis", []),
}
self._prds[prd_id] = prd_version
n_paras = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "para")
n_tables = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "table")
logger.info("[UPLOAD] %.4s 解析完成: %d章节 %d段落 %d表格 %d图片 → %s",
prd_id, len(parsed.get("sections", [])), n_paras, n_tables,
len(parsed.get("image_analysis", [])), parsed_path)
return prd_version
async def upload_quick(self, filename: str, content: bytes) -> dict:
"""Quick save — returns immediately, parsing happens in background."""
prd_id = str(uuid.uuid4())[:8]
upload_dir = self.output_dir / prd_id
os.makedirs(upload_dir, exist_ok=True)
filepath = upload_dir / filename
with open(filepath, "wb") as f:
f.write(content)
logger.info("[UPLOAD] %.4s 快速保存: %s (%.1f KB)", prd_id, filename, len(content) / 1024)
prd_version = {
"prd_id": prd_id, "filename": filename,
"uploaded_at": datetime.now(timezone.utc).isoformat(),
"status": "parsing", "full_text": "", "sections": [], "images": [],
"parsed_path": "", "filepath": str(filepath),
}
self._prds[prd_id] = prd_version
return prd_version
async def parse_async(self, prd_id: str, filename: str, content: bytes):
"""Full parse (text + images + vision) — called in background thread."""
basename = Path(filename).stem
ext = Path(filename).suffix.lower()
upload_dir = self.output_dir / prd_id
filepath = upload_dir / filename
t0 = time.time()
# Parse
if ext in (".txt", ".md"):
text = content.decode("utf-8", errors="replace")
sections = self._parse_markdown_sections(text)
parsed = {"source": str(filepath), "sections": sections,
"image_sources": {}, "image_analysis": [], "full_text": text}
elif ext == ".docx":
parsed = await self._parse_docx(str(filepath), upload_dir)
elif ext == ".pdf":
parsed = await self._parse_pdf(str(filepath), upload_dir)
else:
self.set_status(prd_id, "error", f"Unsupported format: {ext}")
return
# Save parsed
parsed_path = upload_dir / f"{basename}_parsed.json"
with open(parsed_path, "w", encoding="utf-8") as f:
json.dump(parsed, f, ensure_ascii=False, indent=2)
# Update in-memory
prd = self._prds.get(prd_id, {})
prd.update({
"status": "ready", "parsed_path": str(parsed_path),
"full_text": parsed.get("full_text", ""),
"sections": parsed.get("sections", []),
"images": parsed.get("image_analysis", []),
})
n_paras = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "para")
n_tables = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "table")
logger.info("[UPLOAD] %.4s 后台解析完成: %d章节 %d段落 %d表格 %d图片, %.1fs",
prd_id, len(parsed.get("sections", [])), n_paras, n_tables,
len(parsed.get("image_analysis", [])), time.time() - t0)
def set_status(self, prd_id: str, status: str, error: str = ""):
if prd_id in self._prds:
self._prds[prd_id]["status"] = status
if error:
self._prds[prd_id]["error"] = error
async def get_status(self, prd_id: str) -> dict | None:
prd = self._prds.get(prd_id)
if not prd:
return None
return {"prd_id": prd_id, "status": prd.get("status", "ready"),
"error": prd.get("error", ""),
"sections_count": len(prd.get("sections", [])),
"images_count": len(prd.get("images", []))}
async def get_prd(self, prd_id: str) -> dict | None:
return self._prds.get(prd_id)
# ---- internal parsers ----
async def _parse_docx(self, filepath: str, output_dir: Path) -> dict:
word = WordParser(filepath)
sections, image_sources = word.extract_sections()
full_text = word.extract_full_text()
images_dir = output_dir / "images"
images = word.extract_images(str(images_dir))
image_analysis = []
n_chart_images = 0
if images:
parser = ImageParser()
logger.info("[IMAGE] %.4s 发现 %d 张图片,开始视觉分析...", output_dir.parent.name, len(images))
for i, img in enumerate(images):
t_img = time.time()
logger.info("[IMAGE] [%d/%d] rid=%s, 调用 Qwen VL...", i + 1, len(images), img["rid"])
result = parser.parse_image(img["path"])
if result is None:
result = {"type": "other", "description": ""}
result["rid"] = img["rid"]
result["path"] = img["path"]
if img["rid"] in image_sources:
result["context"] = image_sources[img["rid"]]
image_analysis.append(result)
if result.get("type") in ("flowchart", "architecture", "state", "sequence", "activity"):
n_chart_images += 1
logger.info("[IMAGE] [%d/%d] rid=%s 完成,类型=%s (%.1fs)",
i + 1, len(images), img["rid"], result.get("type", "?"), time.time() - t_img)
if i < len(images) - 1:
time.sleep(RATE_LIMIT_DELAY)
usg = parser.usage
logger.info("[IMAGE] %.4s 图片分析完成: %d张, 图表类%d张, tokens(p=%d c=%d t=%d)",
output_dir.parent.name, len(images), n_chart_images,
usg["prompt_tokens"], usg["completion_tokens"], usg["total_tokens"])
else:
logger.info("[IMAGE] %.4s 文档无图片", output_dir.parent.name)
return {
"source": filepath, "sections": sections,
"image_sources": image_sources, "image_analysis": image_analysis,
"full_text": full_text,
}
def _parse_markdown_sections(self, text: str) -> list[dict]:
import re
sections: list[dict] = []
current_source = "正文"
current_blocks: list[dict] = []
para_idx = 0
for line in text.split("\n"):
stripped = line.strip()
if not stripped:
continue
heading_match = re.match(r"^(#{1,6})\s+(.+)", stripped)
if heading_match:
if current_blocks:
sections.append({"source": current_source, "blocks": current_blocks, "images": []})
current_blocks = []
level = len(heading_match.group(1))
current_source = f"{'#' * level} {heading_match.group(2)}"
continue
if "|" in stripped and stripped.count("|") >= 2:
cells = [c.strip() for c in stripped.split("|") if c.strip()]
if cells:
current_blocks.append({
"type": "table",
"table": len([b for b in current_blocks if b["type"] == "table"]) + 1,
"headers": cells,
"rows": [{"columns": [{"name": c, "row": 1, "col": i + 1, "text": c} for i, c in enumerate(cells)]}],
})
continue
para_idx += 1
current_blocks.append({"type": "para", "index": para_idx, "text": stripped})
if current_blocks:
sections.append({"source": current_source, "blocks": current_blocks, "images": []})
return sections
async def _parse_pdf(self, filepath: str, output_dir: Path) -> dict:
from PyPDF2 import PdfReader
reader = PdfReader(filepath)
full_text_parts = []
for i, page in enumerate(reader.pages):
text = page.extract_text()
if text:
full_text_parts.append(text)
full_text = "\n".join(full_text_parts)
sections = [{"source": "PDF 内容", "blocks": [{"type": "para", "index": i + 1, "text": t} for i, t in enumerate(full_text_parts)], "images": []}]
return {"source": filepath, "sections": sections, "image_sources": {}, "image_analysis": [], "full_text": full_text}
prd_service = PRDService()
+255
View File
@@ -0,0 +1,255 @@
# Microsoft Word (.docx) parser — extracts structured sections with typed blocks,
# image markers, and detailed location metadata for downstream IR generation.
import logging
import os
from docx import Document
from docx.table import Table
from docx.text.paragraph import Paragraph
logger = logging.getLogger(__name__)
IMAGE_EXT = {
"image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif",
"image/bmp": ".bmp", "image/tiff": ".tiff", "image/webp": ".webp",
"image/x-emf": ".emf", "image/x-wmf": ".wmf", "image/svg+xml": ".svg",
}
HEADER_CELL_MAX_LEN = 20
WML_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
DRAW_NS = "http://schemas.openxmlformats.org/drawingml/2006/main"
REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
class WordParser:
"""Parse a .docx file — extract images, split body into sections.
Sections contain typed blocks (para/table) with structured table columns
and IMAGE:rid markers, matching the reference document analyzer format.
Usage::
parser = WordParser("doc.docx")
parser.extract_images("images/")
sections, image_sources = parser.extract_sections()
"""
def __init__(self, docx_path: str):
if not os.path.isfile(docx_path):
raise FileNotFoundError(f"Document not found: {docx_path}")
self._doc = Document(docx_path)
# ---- public API ---------------------------------------------------------
def extract_images(self, images_dir: str) -> list[dict]:
"""Save all images to *images_dir*. Returns [{rid, path}, ...]."""
os.makedirs(images_dir, exist_ok=True)
images: list[dict] = []
for rel in self._doc.part.rels.values():
if "image" not in rel.reltype:
continue
ext = IMAGE_EXT.get(rel.target_part.content_type, ".png")
name = f"image_{rel.rId}{ext}"
path = os.path.join(images_dir, name)
with open(path, "wb") as f:
f.write(rel.target_part.blob)
images.append({"rid": rel.rId, "path": path})
return images
def extract_sections(self) -> tuple[list[dict], dict[str, dict]]:
"""Walk document body and split into sections by heading.
Returns:
*sections* — [{source, blocks, images}, ...]
Each block is {type, index, text} (paragraph) or
{type, table, headers, rows} (table with structured columns).
*image_sources* — rid → {section, table?, row?, column?, name?}
"""
sections: list[dict] = []
current_source = ""
blocks: list[dict] = []
section_images: list[str] = []
image_sources: dict[str, dict] = {}
para_idx = 0
tbl_idx = 0
for child in self._doc.element.body:
tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
if tag == "p":
para = Paragraph(child, self._doc)
# Heading detection
if self._heading_level(para) is not None:
heading_text = para.text.strip()
if heading_text:
# Flush previous section
if blocks or section_images:
sections.append({
"source": current_source,
"blocks": blocks,
"images": list(section_images),
})
blocks = []
section_images = []
para_idx = 0
tbl_idx = 0
current_source = heading_text
continue
text = para.text.strip()
# Scan for inline images — append [[IMAGE:rid]] markers
for run in para.runs:
for rid in self._images_in(run._element):
text += f" [[IMAGE:{rid}]]"
section_images.append(rid)
image_sources[rid] = {"section": current_source}
if text.strip():
blocks.append({"type": "para", "index": para_idx + 1, "text": text.strip()})
para_idx += 1
elif tag == "tbl":
tbl_idx += 1
table = Table(child, self._doc)
# Collect all rows with cell text and embedded images
all_rows: list[list[str]] = []
all_images: list[list[list[str]]] = [] # row → col → [rids]
for row in table.rows:
row_texts: list[str] = []
row_cell_images: list[list[str]] = []
for cell in row.cells:
cell_text = cell.text.strip()
cell_imgs: list[str] = []
for cp in cell.paragraphs:
for run in cp.runs:
for rid in self._images_in(run._element):
cell_imgs.append(rid)
for rid in cell_imgs:
cell_text += f" [[IMAGE:{rid}]]"
section_images.append(rid)
row_texts.append(cell_text.strip())
row_cell_images.append(cell_imgs)
if any(row_texts) or any(row_cell_images):
all_rows.append(row_texts)
all_images.append(row_cell_images)
if len(all_rows) >= 2:
# Header heuristic: first row is header if all cells are short
first_row = all_rows[0]
has_header = all(len(c) < HEADER_CELL_MAX_LEN for c in first_row)
if has_header:
headers = first_row
data_rows_slice = zip(all_rows[1:], all_images[1:])
else:
headers = [f"{ci + 1}" for ci in range(len(first_row))]
data_rows_slice = zip(all_rows, all_images)
data_rows: list[dict] = []
for ri, (row_data, row_imgs) in enumerate(data_rows_slice):
columns: list[dict] = []
max_cols = max(len(headers), len(row_data))
for ci in range(max_cols):
hdr = headers[ci] if ci < len(headers) else ""
txt = row_data[ci] if ci < len(row_data) else ""
columns.append({
"name": hdr,
"row": ri + 1,
"col": ci + 1,
"text": txt,
})
# Register image sources with structured location
imgs = row_imgs[ci] if ci < len(row_imgs) else []
for rid in imgs:
image_sources[rid] = {
"section": current_source,
"table": tbl_idx,
"row": ri + 1,
"column": ci + 1,
"name": hdr,
}
data_rows.append({"columns": columns})
blocks.append({
"type": "table",
"table": tbl_idx,
"headers": headers,
"rows": data_rows,
})
elif all_rows:
# Degenerate table — treat as plain rows
for ri, row_data in enumerate(all_rows):
row_text = " | ".join(row_data)
if row_text.strip():
blocks.append({
"type": "para",
"index": para_idx + 1,
"text": row_text,
})
para_idx += 1
# Flush final section
if blocks or section_images:
sections.append({
"source": current_source,
"blocks": blocks,
"images": list(section_images),
})
return sections, image_sources
def extract_full_text(self) -> str:
"""Return all text content as a single string."""
lines = []
for para in self._doc.paragraphs:
text = para.text.strip()
if text:
if self._heading_level(para) is not None:
lines.append(f"\n## {text}\n")
else:
lines.append(text)
for table in self._doc.tables:
for row in table.rows:
lines.append(" | ".join(cell.text.strip() for cell in row.cells))
return "\n".join(lines)
# ---- internals ----------------------------------------------------------
def _heading_level(self, para: Paragraph) -> int | None:
"""Heading level 1-9, or None if not a heading."""
if para.style and para.style.name:
name = para.style.name
for prefix in ("Heading", "标题"):
if name.startswith(prefix):
try:
return int(name.split()[-1])
except (ValueError, IndexError):
pass
pPr = para._element.find(f"{{{WML_NS}}}pPr")
if pPr is not None:
ol = pPr.find(f"{{{WML_NS}}}outlineLvl")
if ol is not None:
val = ol.get(f"{{{WML_NS}}}val")
if val is not None:
try:
return int(val) + 1
except ValueError:
pass
return None
def _images_in(self, element) -> list[str]:
"""Return rId values for drawings embedded in *element*."""
rids: list[str] = []
for drawing in element.findall(f".//{{{WML_NS}}}drawing"):
blip = drawing.find(f".//{{{DRAW_NS}}}blip")
if blip is not None:
rid = blip.get(f"{{{REL_NS}}}embed")
if rid:
rids.append(rid)
return rids
+30
View File
@@ -0,0 +1,30 @@
{persona}
{principles}
## 你的能力
你运行在 ZeekerWatchman 平台中,是一个测试用例管理助手。你可以帮助用户:
1. 分析 PRD 文档,提取功能点
2. 审查和修改 IR(中间表示)
3. 生成和优化测试用例
4. 回答关于测试设计的问题
## 当前左侧面板状态
{page_context}
## 左侧数据
{data_context}
## 操作指令
{action_format}
### IR 规则 ID 格式
IR 规则 ID 格式为 `IR-{FEATURE}-{section}-{NNN}`,如 `IR-ZWM-4.2.1-001`
测试用例 ID 格式为 `TC-{FEATURE}-{NNN}`,如 `TC-ZWM-001`
### 操作格式
当需要让系统执行操作时,在回复末尾附加 action JSON 块:
```action
{{"action": "<action_name>", ...}}
```
每次最多 3 个操作。无操作时不输出。
@@ -0,0 +1,56 @@
你是一个产品需求分析师。你的任务是从 PRD 文档中提取"语义索引"——一份结构化的功能清单,而不是逐字翻译。
## 文档结构说明
下面是一份文档的解析结果,包含:
1. **sections**:按章节组织的混合内容(段落 + 表格),每个 section 有 `source`(章节标题)、`blocks``para` 文本段落和 `table` 结构表格)、`images`(引用的图片 ID 列表)
2. **image_analysis**:文档中图片的程序化分析结果,包含图片类型(flowchart/architecture/state/sequence/activity/other)和文字描述
3. **resolved_conflicts**:文档中图文冲突的仲裁结果(如有)
## 文档全文
{document_json}
## 你的任务
阅读整份文档后,输出一份 **语义索引 JSON**,包含:
### 1. feature_name
从文档中识别的主要功能名称
### 2. concepts
文档中定义或使用的关键概念列表。每个概念包含:
- `name`:概念的标准名称
- `aliases`:同义词/别名列表
- `defined_in`:定义该概念的章节号列表
### 3. function_units
文档中描述的所有主要功能行为的列表。**每个 function_unit 对应一条完整的"如果...则..."规则**。每个 function unit 包含:
- `unit_id`:唯一标识,格式 "FU-001", "FU-002"...
- `name`:简短名称
- `description`1-3 句描述该规则的行为
- `sources`:该规则在文档中的来源锚点列表,每项包含:
- `section`:章节号
- `type`:来源类型,"table"/"para"/"logic_tree"/"image"
- `row`:如果是表格行(从 1 开始)
- `text_snippet`:前 200 字的关键文字
- `image_id`:如果是图片来源,填写图片 rId
## 关键要求
1. **必须覆盖所有图片分析中的逻辑**:图片描述中的每个行为都要出现在某个 function_unit 中
2. **必须覆盖表格中的所有规则**:表格中列出的每条规则都要有对应的 function_unit
3. **区分不同的应用场景和状态**:不同前置条件/状态下的行为分别建模
4. **包含开关/状态切换**:功能开关"开启"和"关闭"两种状态下的行为都要覆盖
## 输出格式
**只输出 JSON,不要有 markdown 代码块标记或其他文字**:
{
"feature_name": "...",
"concepts": [ ... ],
"function_units": [ ... ]
}
@@ -0,0 +1,81 @@
你是一个需求分析专家。你的任务是基于给定的精准上下文包,为单个功能单元(Function Unit)提取详细的 **IR 规则(Intermediate Representation Rule**。
## 上下文
下面是一个功能单元的精准上下文包,包含了从原始需求文档中提取的相关文字、表格和逻辑:
### 功能单元概要
- **unit_id**: {unit_id}
- **unit_name**: {unit_name}
- **unit_description**: {unit_description}
### 相关文字段落
{texts}
### 相关表格
{tables}
### 相关图片分析
{images}
### 冲突仲裁(如有)
{resolved_conflicts}
## IR Schema
你需要为这个功能单元输出一个 **规则数组(rules)**。每条规则遵循以下 schema:
```json
{{
"description": "用完整的中文自然语言描述该规则的触发条件和行为,一句话概括",
"priority": "P0",
"sources": [
{{"type": "table", "section": "...", "row": 2, "text_snippet": "..."}},
{{"type": "image", "image_id": "rId16", "description": "..."}}
],
"precondition": {{
"app_type": "...",
"app_state": "..."
}},
"trigger": {{
"operator": "AND",
"conditions": [
{{"signal": "车速", "operator": ">=", "value": 15, "unit": "km/h"}},
{{"signal": "持续时间", "operator": ">", "value": 5, "unit": "秒"}}
]
}},
"actions": [
{{"type": "system", "description": "系统执行的自动动作"}},
{{"type": "user_interaction", "description": "对用户的提示", "content": "提示文案内容"}}
]
}}
```
### 字段说明
1. **description**: 完整但简洁地描述整个规则,包含前置条件、触发条件和所有动作
2. **priority**: P0(核心规则)、P1(重要规则)、P2(边界情况)
3. **sources**: 每条规则必须列出所有数据来源,包括引用的表格行和图片
4. **precondition**: 规则生效的前置状态条件。可以是空对象 `{{}}`
5. **trigger**: 触发条件对象:
- `operator`: "AND" 或 "OR"
- `conditions`: 条件数组,每个条件有 `signal`、`operator`、`value`,可选 `unit`
6. **actions**: 每个动作有 `type`"system"/"user_interaction")和 `description`。用户可见交互包含 `content` 字段
## 关键要求
1. **信号和数值必须精确**:不写"车速超过阈值",写 `"车速 >= 15 km/h"`
2. **条件必须完整**:文档中的所有触发条件都要出现在 trigger.conditions 中
3. **动作类型区分**:系统行为用 "system",用户可见交互用 "user_interaction"
4. **多条规则**:如果一个功能单元包含多个独立的行为分支,输出多条规则分别描述
## 输出格式
**只输出 JSON 数组,不要有任何其他文字或 markdown 标记**
[
{{ ... }},
{{ ... }}
]
即使只有一个规则,也必须用数组格式 `[...]`。
+108
View File
@@ -0,0 +1,108 @@
# Skill 管理服务:扫描、验证、列出所有可用 Skill
import json
import logging
from pathlib import Path
import yaml
from server.config import settings
logger = logging.getLogger("testflow")
REQUIRED_SKILL_FILES = ["skill.yaml", "ir_schema.json", "extract_ir_prompt.j2", "gen_cases_prompt.j2"]
class SkillManager:
"""Discover and validate skill packages from .zeekerwatchmen/skills/."""
def __init__(self):
self._skills_dir = settings.SKILLS_DIR
self._cache: dict[str, dict] = {}
self._scan()
def _scan(self):
"""Scan the skills directory and cache valid skills."""
self._cache = {}
if not self._skills_dir.exists():
logger.warning("[SKILL] s directory not found: %s", self._skills_dir)
return
for skill_dir in sorted(self._skills_dir.iterdir()):
if not skill_dir.is_dir():
continue
skill_name = skill_dir.name
skill = self._load_skill(skill_name)
if skill:
self._cache[skill_name] = skill
logger.info("[SKILL] 加载: %s (v%s)", skill_name, skill.get("version", "?"))
logger.info("[SKILL] 发现 %d skills: %s", len(self._cache), list(self._cache.keys()))
def _load_skill(self, skill_name: str) -> dict | None:
"""Load and validate a single skill package."""
skill_dir = self._skills_dir / skill_name
# Check required files
missing = []
for filename in REQUIRED_SKILL_FILES:
if not (skill_dir / filename).exists():
missing.append(filename)
if missing:
logger.warning("[SKILL] '%s' missing files: %s", skill_name, missing)
return None
# Load metadata
try:
with open(skill_dir / "skill.yaml", "r", encoding="utf-8") as f:
meta = yaml.safe_load(f) or {}
except yaml.YAMLError as e:
logger.error("[SKILL] '%s': invalid skill.yaml: %s", skill_name, e)
return None
# Load schema
try:
with open(skill_dir / "ir_schema.json", "r", encoding="utf-8") as f:
schema = json.load(f)
except (json.JSONDecodeError, IOError) as e:
logger.error("[SKILL] '%s': invalid ir_schema.json: %s", skill_name, e)
return None
return {
"name": skill_name,
"display_name": meta.get("display_name", skill_name),
"description": meta.get("description", ""),
"version": meta.get("version", "0.0.0"),
"domain": meta.get("domain", "general"),
"author": meta.get("author", ""),
"auto_match": meta.get("auto_match", {}),
"schema_fields": list(schema.get("properties", {}).get("features", {}).get("items", {}).get("properties", {}).keys()),
}
def list_skills(self) -> list[dict]:
"""Return metadata for all discovered skills."""
return [
{
"name": s["name"],
"display_name": s["display_name"],
"description": s["description"],
"version": s["version"],
"domain": s["domain"],
}
for s in self._cache.values()
]
def get_skill(self, name: str) -> dict | None:
"""Get a specific skill's full metadata."""
return self._cache.get(name)
def reload(self):
"""Re-scan the skills directory (for hot-reload)."""
logger.info("[SKILL] 热重载 skills...")
self._cache = {}
self._scan()
skill_manager = SkillManager()
@@ -0,0 +1,66 @@
# 用例导出服务:将用例集转换为 YAML / CSV 格式
import csv
import io
import logging
import yaml
from server.services.testcase_engine.generator import tc_generator
logger = logging.getLogger(__name__)
class TestCaseExporter:
"""Export test case sets to YAML, CSV, or structured data."""
async def export(self, tc_set_id: str, fmt: str) -> tuple[bytes | str, str]:
"""Export testcases in specified format. Returns (content, mime_type, filename)."""
tc_set = await tc_generator.get_tc_set(tc_set_id)
if not tc_set:
raise ValueError(f"Test case set not found: {tc_set_id}")
cases = tc_set.get("cases", [])
if fmt == "yaml":
return self._to_yaml(cases), "application/x-yaml", f"testcases_{tc_set_id}.yaml"
elif fmt == "csv":
return self._to_csv(cases), "text/csv", f"testcases_{tc_set_id}.csv"
elif fmt == "json":
import json
return json.dumps(tc_set, ensure_ascii=False, indent=2), "application/json", f"testcases_{tc_set_id}.json"
else:
raise ValueError(f"Unsupported export format: {fmt}")
def _to_yaml(self, cases: list[dict]) -> str:
"""Convert cases to YAML string."""
output = {
"testcases": cases,
"metadata": {
"total": len(cases),
"exported_format": "yaml",
},
}
return yaml.dump(output, allow_unicode=True, sort_keys=False, default_flow_style=False)
def _to_csv(self, cases: list[dict]) -> str:
"""Convert cases to CSV string."""
if not cases:
return ""
# Flatten tags for CSV
flattened = []
for c in cases:
row = {k: v for k, v in c.items() if k != "tags"}
row["tags"] = ";".join(c.get("tags", []))
flattened.append(row)
output = io.StringIO()
fieldnames = ["id", "module", "feature", "case_title", "preconditions", "steps", "expected_result", "priority", "tags"]
writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore")
writer.writeheader()
writer.writerows(flattened)
return output.getvalue()
tc_exporter = TestCaseExporter()
@@ -0,0 +1,212 @@
# 测试用例生成器:基于 IR 生成测试用例集
import json
import logging
import uuid
from datetime import datetime, timezone
from server.config import settings
from server.core.llm_provider.router import router
from server.services.ir_engine.generator import ir_generator
logger = logging.getLogger("testflow")
DEFAULT_GEN_PROMPT = """你是一位资深的软件测试工程师。基于以下中间表示(IR),请生成完整的测试用例集。
测试设计原则:
1. 每个功能点至少覆盖一条正向用例和一条异常用例
2. 对于 P0 功能点,额外覆盖边界值测试
3. 用例步骤使用 Given-When-Then 结构
4. 预期结果必须明确可验证
输出格式:纯 JSON 数组,每个测试用例包含以下字段:
- id: 唯一标识符(字符串)
- module: 模块名
- feature: 功能点名称
- case_title: 用例标题
- preconditions: 前置条件
- steps: 测试步骤(Given-When-Then 格式)
- expected_result: 预期结果
- priority: 用例优先级 (P0/P1/P2)
- tags: 标签数组(如:["正向", "异常", "边界"]
## IR 内容:
{{ ir_content }}
请只输出 JSON 数组,不要用 ```json 代码块包裹。"""
class TestCaseGenerator:
"""Generate test case sets from IR YAML using LLM."""
def __init__(self):
self._tc_store: dict[str, dict] = {}
async def generate(self, ir_id: str, skill_name: str = "default") -> dict:
"""Generate test cases from an IR version."""
# Get IR content
ir_version = await ir_generator.get_ir(ir_id)
if not ir_version:
return {"error": f"IR version not found: {ir_id}", "cases": []}
# Handle both old YAML and new pipeline JSON formats
ir_content = ir_version.get("yaml_content", "")
ir_json = ir_version.get("ir_json")
# If we have the new pipeline format (rules array), convert to testcases directly
if ir_json and ir_json.get("rules"):
logger.info("[TC] 从 %d 条 IR 规则转换测试用例...", len(ir_json["rules"]))
cases = self._rules_to_cases(ir_json["rules"], ir_json.get("feature", ""))
tc_set_id = str(uuid.uuid4())[:8]
tc_set = self._store_tc(tc_set_id, ir_id, cases)
# Log distribution
p0 = sum(1 for c in cases if c.get("priority") == "P0")
p1 = sum(1 for c in cases if c.get("priority") == "P1")
logger.info("[TC] ✅ 生成 %d 条用例 (P0:%d P1:%d P2:%d)",
len(cases), p0, p1, len(cases) - p0 - p1)
return tc_set
# Fallback: use LLM with skill prompt
if not ir_content:
return {"error": "IR has no content", "cases": [], "tc_set_id": ""}
# Load skill prompt
skill = ir_generator.load_skill(skill_name)
prompt_template = skill.get("gen_cases_prompt", DEFAULT_GEN_PROMPT)
prompt = prompt_template.replace("{{ ir_content }}", ir_content)
logger.info("[TC] 调用 LLM 生成测试用例 for IR %s with skill %s", ir_id, skill_name)
client = router.get_text_client()
try:
raw = client.chat(
model=router.text_model,
messages=[{"role": "user", "content": prompt}],
temperature=0.0,
)
except Exception as e:
logger.error("Test case generation failed: %s", e)
return {"error": str(e), "cases": [], "tc_set_id": ""}
# Parse JSON response
cases = self._parse_json(raw)
# Create test case set
tc_set_id = str(uuid.uuid4())[:8]
tc_set = {
"tc_set_id": tc_set_id,
"ir_id": ir_id,
"cases": cases,
"case_count": len(cases),
"created_at": datetime.now(timezone.utc).isoformat(),
}
self._tc_store[tc_set_id] = tc_set
# Save to output
output_dir = settings.OUTPUT_DIR / ir_version.get("prd_id", "unknown") / "tc_sets"
output_dir.mkdir(parents=True, exist_ok=True)
tc_path = output_dir / f"tc_{tc_set_id}.json"
with open(tc_path, "w", encoding="utf-8") as f:
json.dump(tc_set, f, ensure_ascii=False, indent=2)
usg = client.usage
logger.info("Test cases generated: id=%s, count=%d, tokens: prompt=%d completion=%d total=%d",
tc_set_id, len(cases), usg["prompt_tokens"], usg["completion_tokens"], usg["total_tokens"])
return tc_set
async def get_tc_set(self, tc_set_id: str) -> dict | None:
"""Get a test case set by ID."""
return self._tc_store.get(tc_set_id)
@staticmethod
def _parse_json(raw: str) -> list[dict]:
"""Parse JSON from LLM response, handling code fences."""
text = raw.strip()
if text.startswith("```json") or text.startswith("```"):
first_nl = text.find("\n")
if first_nl != -1:
text = text[first_nl + 1:]
if text.endswith("```"):
text = text[:-3]
text = text.strip()
try:
parsed = json.loads(text)
if isinstance(parsed, list):
return parsed
if isinstance(parsed, dict) and "cases" in parsed:
return parsed["cases"]
return [parsed] if isinstance(parsed, dict) else []
except json.JSONDecodeError:
logger.warning("Failed to parse test case JSON, returning empty")
return []
def _rules_to_cases(self, rules: list[dict], feature: str) -> list[dict]:
"""Convert pipeline IR rules to test case format. TC IDs: TC-{feature_id}-{seq}"""
import re
feature_id = re.sub(r'[^A-Z]', '', feature.upper())[:6] or "FEAT"
if len(feature_id) < 2:
feature_id = (feature_id + "FEAT")[:4]
cases = []
for i, rule in enumerate(rules):
tc_id = f"TC-{feature_id}-{i+1:03d}"
desc = rule.get("description", "")
# Build steps string from trigger + actions
trigger = rule.get("trigger", {})
conditions = trigger.get("conditions", [])
cond_texts = []
for c in conditions:
unit = f" {c.get('unit', '')}" if c.get('unit') else ""
cond_texts.append(f"{c.get('signal','?')} {c.get('operator','?')} {c.get('value','?')}{unit}")
given = "Given " + (", ".join(cond_texts) if cond_texts else "触发条件满足")
actions = rule.get("actions", [])
action_texts = []
for a in actions:
if a.get("type") == "user_interaction":
action_texts.append(f"{a.get('description','')}('{a.get('content','')}')")
else:
action_texts.append(a.get("description", ""))
then = "Then " + ("; ".join(action_texts) if action_texts else "执行动作")
steps = f"{given}\nWhen 条件触发\n{then}"
# Determine tags
tags = ["正向"]
priority = rule.get("priority", "P2")
if priority == "P0":
tags.append("冒烟")
precond = rule.get("precondition", {})
if precond:
tags.append(f"{precond.get('app_type', '')} {precond.get('app_state', '')}".strip())
cases.append({
"id": tc_id,
"ir_rule_id": rule.get("rule_id", ""),
"module": feature,
"feature": rule.get("description", desc)[:40],
"case_title": desc[:60],
"preconditions": json.dumps(precond, ensure_ascii=False) if precond else "",
"steps": steps,
"expected_result": action_texts[-1] if action_texts else desc,
"priority": priority,
"tags": [t for t in tags if t],
})
return cases
def _store_tc(self, tc_set_id: str, ir_id: str, cases: list[dict]) -> dict:
"""Store a test case set and return its metadata."""
tc_set = {
"tc_set_id": tc_set_id,
"ir_id": ir_id,
"cases": cases,
"case_count": len(cases),
"created_at": datetime.now(timezone.utc).isoformat(),
}
self._tc_store[tc_set_id] = tc_set
return tc_set
tc_generator = TestCaseGenerator()