init the project
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
# IR 版本差异对比服务
|
||||
|
||||
import logging
|
||||
|
||||
import yaml
|
||||
|
||||
from server.core.utils.diff import compute_diff
|
||||
from server.services.ir_engine.generator import ir_generator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IRDiffService:
|
||||
"""Compare two IR versions and return a structured diff."""
|
||||
|
||||
async def diff(self, ir_id_a: str, ir_id_b: str) -> dict:
|
||||
"""Compute diff between two IR versions."""
|
||||
ir_a = await ir_generator.get_ir(ir_id_a)
|
||||
ir_b = await ir_generator.get_ir(ir_id_b)
|
||||
|
||||
if not ir_a or not ir_b:
|
||||
return {"error": "One or both IR versions not found", "diff": {}}
|
||||
|
||||
yaml_a = ir_a.get("yaml_content", "")
|
||||
yaml_b = ir_b.get("yaml_content", "")
|
||||
|
||||
try:
|
||||
data_a = yaml.safe_load(yaml_a)
|
||||
data_b = yaml.safe_load(yaml_b)
|
||||
except yaml.YAMLError as e:
|
||||
return {"error": f"YAML parsing error: {e}", "diff": {}}
|
||||
|
||||
diff_result = compute_diff(data_a or {}, data_b or {})
|
||||
logger.info("IR diff: %d added, %d removed, %d modified",
|
||||
len(diff_result.get("added", [])),
|
||||
len(diff_result.get("removed", [])),
|
||||
len(diff_result.get("modified", [])))
|
||||
|
||||
return {
|
||||
"ir_id_a": ir_id_a,
|
||||
"ir_id_b": ir_id_b,
|
||||
"diff": diff_result,
|
||||
}
|
||||
|
||||
|
||||
ir_diff = IRDiffService()
|
||||
@@ -0,0 +1,76 @@
|
||||
# IR Generator — delegates to the 3-stage pipeline
|
||||
|
||||
import logging
|
||||
|
||||
from server.services.ir_engine.pipeline import ir_pipeline
|
||||
from server.services.prd_manager.service import prd_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IRGenerator:
|
||||
"""IR generator that runs the full 3-stage pipeline on parsed PRD documents."""
|
||||
|
||||
async def generate(self, prd_id: str, prd_text: str = "", skill_name: str = "default") -> dict:
|
||||
"""Run the pipeline on a parsed PRD document."""
|
||||
# Get the parsed document from the PRD service
|
||||
prd = await prd_service.get_prd(prd_id)
|
||||
if not prd:
|
||||
return {"error": f"PRD not found: {prd_id}", "ir_id": ""}
|
||||
|
||||
# Fetch full parsed data from disk if available
|
||||
parsed_doc = self._load_parsed(prd)
|
||||
if not parsed_doc:
|
||||
# Fallback: build minimal doc from PRD text
|
||||
parsed_doc = {
|
||||
"source": "",
|
||||
"sections": [{"source": "正文", "blocks": [{"type": "para", "index": 1, "text": prd.get("full_text", "")}], "images": []}],
|
||||
"image_sources": {},
|
||||
"image_analysis": [],
|
||||
"resolved_conflicts": [],
|
||||
}
|
||||
|
||||
result = await ir_pipeline.run(prd_id, parsed_doc)
|
||||
return {
|
||||
"ir_id": result["ir_id"],
|
||||
"prd_id": result["prd_id"],
|
||||
"yaml_content": result["yaml_content"],
|
||||
"ir_json": result["ir_json"],
|
||||
"audit": result["audit"],
|
||||
"audit_report": result["audit_report"],
|
||||
"skill_used": skill_name,
|
||||
"created_at": result["created_at"],
|
||||
"pipeline_stats": result["pipeline_stats"],
|
||||
}
|
||||
|
||||
async def get_ir(self, ir_id: str) -> dict | None:
|
||||
return await ir_pipeline.get_ir(ir_id)
|
||||
|
||||
def _load_parsed(self, prd: dict) -> dict | None:
|
||||
"""Load the full parsed JSON from disk if available."""
|
||||
import json
|
||||
parsed_path = prd.get("parsed_path", "")
|
||||
if parsed_path:
|
||||
try:
|
||||
with open(parsed_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Build from in-memory data
|
||||
if prd.get("sections") or prd.get("full_text"):
|
||||
sections = prd.get("sections", [])
|
||||
if not sections and prd.get("full_text"):
|
||||
sections = [{"source": "正文", "blocks": [{"type": "para", "index": 1, "text": prd["full_text"]}], "images": []}]
|
||||
return {
|
||||
"source": "",
|
||||
"sections": sections,
|
||||
"image_sources": {},
|
||||
"image_analysis": prd.get("images", []),
|
||||
"resolved_conflicts": [],
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
ir_generator = IRGenerator()
|
||||
@@ -0,0 +1,753 @@
|
||||
# 3-Stage IR Generation Pipeline
|
||||
#
|
||||
# Stage 1: Semantic Index — full document → function_units + concepts
|
||||
# Stage 2: Per-Unit IR Extraction — precision context → detailed IR rules
|
||||
# Stage 3: Deterministic Merge & Audit — dedup, assign IDs, coverage report
|
||||
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from server.config import settings
|
||||
from server.core.llm_provider.router import router
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROMPTS_DIR = Path(__file__).parent.parent / "prompts" # services/prompts/
|
||||
MAX_CONCURRENCY = 5
|
||||
|
||||
PASS, WARN, FAIL = "[PASS]", "[WARN]", "[FAIL]"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Stage 1: Semantic Index Generation
|
||||
# =============================================================================
|
||||
|
||||
def format_document_for_prompt(doc: dict) -> str:
|
||||
"""Render the full parsed document as a readable string for the LLM prompt."""
|
||||
lines = ["=== SECTIONS ==="]
|
||||
|
||||
for i, section in enumerate(doc.get("sections", [])):
|
||||
source = section.get("source", f"(无标题-章节{i})")
|
||||
lines.append(f"\n--- Section: {source} ---")
|
||||
|
||||
for block in section.get("blocks", []):
|
||||
if block["type"] == "para":
|
||||
lines.append(f"[段落 {block['index']}] {block['text']}")
|
||||
elif block["type"] == "table":
|
||||
lines.append(f"[表格 {block.get('table', '?')}]")
|
||||
headers = block.get("headers", [])
|
||||
lines.append(f" 表头: {' | '.join(headers)}")
|
||||
for row in block.get("rows", []):
|
||||
cols = row.get("columns", [])
|
||||
cell_texts = []
|
||||
for c in cols:
|
||||
cell_texts.append(f"[行{c.get('row','?')}]{c.get('name','')}: {c.get('text','')}")
|
||||
lines.append(f" {'; '.join(cell_texts)}")
|
||||
|
||||
images = section.get("images", [])
|
||||
if images:
|
||||
lines.append(f" 图片引用: {', '.join(images)}")
|
||||
|
||||
# Image Analysis
|
||||
lines.append("\n\n=== IMAGE_ANALYSIS ===")
|
||||
for img in doc.get("image_analysis", []):
|
||||
rid = img.get("rid", "?")
|
||||
img_type = img.get("type", "?")
|
||||
lines.append(f"\n--- Image: {rid} (type={img_type}) ---")
|
||||
lines.append(f" 描述: {img.get('description', '')[:500]}")
|
||||
|
||||
# Resolved Conflicts
|
||||
conflicts = doc.get("resolved_conflicts", [])
|
||||
if conflicts:
|
||||
lines.append("\n\n=== RESOLVED_CONFLICTS ===")
|
||||
for c in conflicts:
|
||||
lines.append(
|
||||
f" [{c.get('conflict_type','?')}] {c.get('section','?')}: "
|
||||
f"以{c.get('source','?')}为准 — {c.get('correction','')}"
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def extract_json_from_response(text: str, expect_array: bool = False) -> str:
|
||||
"""Robustly extract JSON object or array from LLM response."""
|
||||
open_char = "[" if expect_array else "{"
|
||||
close_char = "]" if expect_array else "}"
|
||||
|
||||
# Try code fence first
|
||||
pattern = rf"```(?:json)?\s*(\[{open_char}[\s\S]*?\{close_char}\])\s*```" if expect_array else rf"```(?:json)?\s*(\{{[\s\S]*?\}})\s*```"
|
||||
m = re.search(pattern, text)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
|
||||
# Find outermost bracket
|
||||
start = text.find(open_char)
|
||||
if start == -1:
|
||||
raise ValueError(f"No JSON {'array' if expect_array else 'object'} found in LLM response")
|
||||
|
||||
depth = 0
|
||||
for i in range(start, len(text)):
|
||||
if text[i] == open_char:
|
||||
depth += 1
|
||||
elif text[i] == close_char:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start: i + 1]
|
||||
|
||||
raise ValueError(f"Unclosed JSON {'array' if expect_array else 'object'} in LLM response")
|
||||
|
||||
|
||||
async def stage1_semantic_index(doc: dict) -> dict:
|
||||
"""Stage 1: Generate semantic index from full document."""
|
||||
logger.info("[PIPELINE] ═══ Stage 1/3: 语义索引 ═══")
|
||||
|
||||
template_path = PROMPTS_DIR / "ir_step1_semantic_index.txt"
|
||||
if not template_path.exists():
|
||||
logger.warning("[PIPELINE] Stage 1 prompt template not found, using fallback")
|
||||
return _fallback_semantic_index(doc)
|
||||
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
formatted_doc = format_document_for_prompt(doc)
|
||||
prompt = template.replace("{document_json}", formatted_doc)
|
||||
|
||||
logger.info("[PIPELINE] Prompt: %d chars (~%d tokens)", len(prompt), len(prompt) // 3)
|
||||
|
||||
client = router.get_text_client()
|
||||
for attempt in range(3):
|
||||
try:
|
||||
raw = client.chat(
|
||||
model=router.text_model,
|
||||
messages=[
|
||||
{"role": "system", "content": "你是一个精确的 JSON 输出引擎。只输出合法的 JSON,不输出任何其他文字。"},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
temperature=0.1,
|
||||
)
|
||||
json_str = extract_json_from_response(raw)
|
||||
result = json.loads(json_str)
|
||||
|
||||
# Validate
|
||||
if "function_units" not in result or not result["function_units"]:
|
||||
raise ValueError("Missing function_units")
|
||||
logger.info("[PIPELINE] ✅ Stage 1 完成: %d function_units, %d concepts",
|
||||
len(result["function_units"]), len(result.get("concepts", [])))
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("[PIPELINE] Stage 1 attempt %d failed: %s", attempt + 1, e)
|
||||
if attempt < 2:
|
||||
time.sleep(2)
|
||||
|
||||
# Fallback
|
||||
logger.warning("[PIPELINE] Stage 1 全部重试失败 exhausted, using fallback")
|
||||
return _fallback_semantic_index(doc)
|
||||
|
||||
|
||||
def _fallback_semantic_index(doc: dict) -> dict:
|
||||
"""Generate a basic semantic index without LLM (for mock/demo)."""
|
||||
units = []
|
||||
unit_idx = 1
|
||||
|
||||
for section in doc.get("sections", []):
|
||||
source = section.get("source", "")
|
||||
for block in section.get("blocks", []):
|
||||
if block["type"] == "para":
|
||||
# Create a function unit from each substantial paragraph
|
||||
text = block.get("text", "")
|
||||
if len(text) > 30:
|
||||
units.append({
|
||||
"unit_id": f"FU-{unit_idx:03d}",
|
||||
"name": text[:60],
|
||||
"description": text[:200],
|
||||
"sources": [{"section": source, "type": "para", "text_snippet": text[:200]}],
|
||||
})
|
||||
unit_idx += 1
|
||||
elif block["type"] == "table":
|
||||
for row in block.get("rows", []):
|
||||
cols = row.get("columns", [])
|
||||
texts = [f"{c.get('name','')}: {c.get('text','')}" for c in cols if c.get('text')]
|
||||
if texts:
|
||||
combined = "; ".join(texts)
|
||||
units.append({
|
||||
"unit_id": f"FU-{unit_idx:03d}",
|
||||
"name": combined[:60],
|
||||
"description": combined[:200],
|
||||
"sources": [{"section": source, "type": "table", "row": row.get("columns", [{}])[0].get("row", 1) if cols else 1, "text_snippet": combined[:200]}],
|
||||
})
|
||||
unit_idx += 1
|
||||
|
||||
return {
|
||||
"feature_name": doc.get("sections", [{}])[0].get("source", "未命名功能"),
|
||||
"concepts": [],
|
||||
"function_units": units,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Stage 2: Per-Unit IR Extraction
|
||||
# =============================================================================
|
||||
|
||||
def build_context_package(fu: dict, doc: dict, image_sources: dict) -> dict:
|
||||
"""Build a precision context package for one function unit."""
|
||||
texts = []
|
||||
tables = []
|
||||
images = []
|
||||
seen_sections = set()
|
||||
|
||||
img_by_rid = {img.get("rid", ""): img for img in doc.get("image_analysis", [])}
|
||||
|
||||
for src in fu.get("sources", []):
|
||||
section_key = src.get("section", "")
|
||||
src_type = src.get("type", "")
|
||||
|
||||
if src_type in ("para", "table") and section_key:
|
||||
if section_key in seen_sections:
|
||||
continue
|
||||
seen_sections.add(section_key)
|
||||
|
||||
# Find matching section
|
||||
for section in doc.get("sections", []):
|
||||
if section.get("source", "") == section_key:
|
||||
for block in section.get("blocks", []):
|
||||
if block["type"] == "para":
|
||||
texts.append({
|
||||
"section": section_key,
|
||||
"text": block["text"],
|
||||
})
|
||||
elif block["type"] == "table":
|
||||
all_rows = []
|
||||
for row in block.get("rows", []):
|
||||
for col in row.get("columns", []):
|
||||
all_rows.append({
|
||||
"row": col.get("row"),
|
||||
"name": col.get("name"),
|
||||
"text": col.get("text"),
|
||||
})
|
||||
# Trim: max 20 cells per table to reduce prompt size
|
||||
tables.append({
|
||||
"section": section_key,
|
||||
"headers": block.get("headers", []),
|
||||
"all_rows": all_rows[:20],
|
||||
"_truncated": len(all_rows) > 20,
|
||||
})
|
||||
break
|
||||
|
||||
if src_type in ("image", "logic_tree"):
|
||||
img_id = src.get("image_id", "")
|
||||
if img_id and img_id in img_by_rid:
|
||||
img = img_by_rid[img_id]
|
||||
images.append({
|
||||
"image_id": img_id,
|
||||
"type": img.get("type", "other"),
|
||||
"description": img.get("description", ""),
|
||||
})
|
||||
|
||||
return {
|
||||
"unit_id": fu["unit_id"],
|
||||
"unit_name": fu.get("name", ""),
|
||||
"unit_description": fu.get("description", ""),
|
||||
"texts": texts,
|
||||
"tables": tables,
|
||||
"images": images,
|
||||
"resolved_conflicts": doc.get("resolved_conflicts", []),
|
||||
}
|
||||
|
||||
|
||||
async def stage2_extract_rules(semantic_index: dict, doc: dict) -> list[dict]:
|
||||
"""Stage 2: Extract detailed IR rules for each function unit."""
|
||||
logger.info("=" * 50)
|
||||
logger.info("[PIPELINE] ═══ Stage 2/3: Per-Unit IR Extraction")
|
||||
|
||||
template_path = PROMPTS_DIR / "ir_step2_extraction.txt"
|
||||
if not template_path.exists():
|
||||
return _fallback_rules(semantic_index)
|
||||
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
function_units = semantic_index.get("function_units", [])
|
||||
image_sources = doc.get("image_sources", {})
|
||||
|
||||
logger.info("Processing %d function units (max concurrency: %d)", len(function_units), MAX_CONCURRENCY)
|
||||
|
||||
# Build context packages
|
||||
packages = []
|
||||
for fu in function_units:
|
||||
pkg = build_context_package(fu, doc, image_sources)
|
||||
packages.append(pkg)
|
||||
|
||||
# Parallel LLM calls
|
||||
fragments = []
|
||||
client = router.get_text_client()
|
||||
|
||||
def extract_one(pkg: dict) -> dict:
|
||||
"""Extract rules for one function unit."""
|
||||
prompt = _build_stage2_prompt(template, pkg)
|
||||
unit_id = pkg["unit_id"]
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
raw = client.chat(
|
||||
model=router.text_model,
|
||||
messages=[
|
||||
{"role": "system", "content": "你是一个精确的 JSON 输出引擎。只输出合法的 JSON 数组,不输出任何其他文字。"},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
temperature=0.1,
|
||||
)
|
||||
json_str = extract_json_from_response(raw, expect_array=True)
|
||||
rules = json.loads(json_str)
|
||||
if not isinstance(rules, list):
|
||||
raise ValueError(f"Expected array, got {type(rules).__name__}")
|
||||
return {"unit_id": unit_id, "unit_name": pkg["unit_name"], "rules": rules}
|
||||
except Exception as e:
|
||||
if attempt < 2:
|
||||
time.sleep(2)
|
||||
else:
|
||||
logger.warning("Unit %s failed after 3 attempts: %s", unit_id, e)
|
||||
return {"unit_id": unit_id, "unit_name": pkg["unit_name"], "rules": [], "error": str(e)}
|
||||
return {"unit_id": unit_id, "unit_name": pkg["unit_name"], "rules": [], "error": "unknown"}
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENCY) as executor:
|
||||
futures = {executor.submit(extract_one, pkg): pkg for pkg in packages}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
result = future.result()
|
||||
fragments.append(result)
|
||||
n_rules = len(result.get("rules", []))
|
||||
logger.info("[PIPELINE] %s: %d rules %s", result["unit_id"], n_rules,
|
||||
"❌" if result.get("error") else "✅")
|
||||
|
||||
fragments.sort(key=lambda f: f["unit_id"])
|
||||
total = sum(len(f.get("rules", [])) for f in fragments)
|
||||
logger.info("[PIPELINE] ✅ Stage 2 完成: %d fragments, %d total rules", len(fragments), total)
|
||||
return fragments
|
||||
|
||||
|
||||
def _build_stage2_prompt(template: str, pkg: dict) -> str:
|
||||
"""Render the stage 2 prompt template with context package data."""
|
||||
# Simple str.replace (avoids format issues with curly braces in JSON)
|
||||
prompt = template
|
||||
prompt = prompt.replace("{unit_id}", pkg["unit_id"])
|
||||
prompt = prompt.replace("{unit_name}", pkg["unit_name"])
|
||||
prompt = prompt.replace("{unit_description}", pkg["unit_description"])
|
||||
prompt = prompt.replace("{texts}", json.dumps(pkg.get("texts", []), ensure_ascii=False, indent=2))
|
||||
prompt = prompt.replace("{tables}", json.dumps(pkg.get("tables", []), ensure_ascii=False, indent=2))
|
||||
prompt = prompt.replace("{images}", json.dumps(pkg.get("images", []), ensure_ascii=False, indent=2))
|
||||
prompt = prompt.replace("{resolved_conflicts}", json.dumps(pkg.get("resolved_conflicts", []), ensure_ascii=False, indent=2))
|
||||
return prompt
|
||||
|
||||
|
||||
def _fallback_rules(semantic_index: dict) -> list[dict]:
|
||||
"""Generate basic rules without LLM."""
|
||||
fragments = []
|
||||
for fu in semantic_index.get("function_units", []):
|
||||
fragments.append({
|
||||
"unit_id": fu["unit_id"],
|
||||
"unit_name": fu.get("name", ""),
|
||||
"rules": [{
|
||||
"description": fu.get("description", ""),
|
||||
"priority": "P2",
|
||||
"sources": fu.get("sources", []),
|
||||
"precondition": {},
|
||||
"trigger": {"operator": "AND", "conditions": []},
|
||||
"actions": [],
|
||||
}],
|
||||
})
|
||||
return fragments
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Stage 3: Merge & Audit
|
||||
# =============================================================================
|
||||
|
||||
def rule_signature(rule: dict) -> str:
|
||||
"""SHA256 signature from trigger + actions for dedup."""
|
||||
trigger = rule.get("trigger", {})
|
||||
actions = rule.get("actions", [])
|
||||
conditions = sorted(trigger.get("conditions", []), key=lambda c: json.dumps(c, sort_keys=True))
|
||||
sorted_actions = sorted(actions, key=lambda a: json.dumps(a, sort_keys=True))
|
||||
sig = json.dumps({"conditions": conditions, "actions": sorted_actions}, ensure_ascii=False, sort_keys=True)
|
||||
return hashlib.sha256(sig.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def stage3_merge_and_audit(fragments: list[dict], semantic_index: dict, doc: dict) -> dict:
|
||||
"""Stage 3: Merge rules, assign IDs, and generate audit report."""
|
||||
logger.info("=" * 50)
|
||||
logger.info("[PIPELINE] ═══ Stage 3/3: Merge & Audit")
|
||||
|
||||
# Merge and dedup
|
||||
sig_map: dict[str, dict] = {}
|
||||
order = []
|
||||
for f in fragments:
|
||||
for rule in f.get("rules", []):
|
||||
sig = rule_signature(rule)
|
||||
if sig in sig_map:
|
||||
existing = sig_map[sig]
|
||||
existing_sources = existing.setdefault("sources", [])
|
||||
for src in rule.get("sources", []):
|
||||
if src not in existing_sources:
|
||||
existing_sources.append(src)
|
||||
if len(rule.get("description", "")) > len(existing.get("description", "")):
|
||||
existing["description"] = rule["description"]
|
||||
else:
|
||||
sig_map[sig] = dict(rule)
|
||||
order.append(sig)
|
||||
|
||||
merged = [sig_map[sig] for sig in order]
|
||||
logger.info("[PIPELINE] 合并: %d → %d rules (dedup)", sum(len(f.get("rules", [])) for f in fragments), len(merged))
|
||||
|
||||
# Assign rule IDs: IR-{feature_id}-{section}-{seq}
|
||||
section_counters = defaultdict(int)
|
||||
feature_name = semantic_index.get("feature_name", "FEATURE")
|
||||
feature_id = re.sub(r'[^A-Z]', '', feature_name.upper())[:6] or "FEAT"
|
||||
if len(feature_id) < 2:
|
||||
feature_id = (feature_id + "FEATURE")[:4]
|
||||
|
||||
for rule in merged:
|
||||
# Extract section from first para/table source, fallback to "0"
|
||||
section = "0"
|
||||
for src in rule.get("sources", []):
|
||||
s = src.get("section", "")
|
||||
if s:
|
||||
# Extract first number pattern like "4.2.1" or "3.1"
|
||||
m = re.search(r'[\d]+(?:\.[\d]+)*', s)
|
||||
if m:
|
||||
section = m.group()
|
||||
break
|
||||
section_counters[section] += 1
|
||||
rule["rule_id"] = f"IR-{feature_id}-{section}-{section_counters[section]:03d}"
|
||||
|
||||
# Build final IR
|
||||
ir_final = {
|
||||
"feature": feature_name,
|
||||
"feature_id": feature_id,
|
||||
"rules": merged,
|
||||
"rule_count": len(merged),
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
# Generate audit report
|
||||
audit_results = _generate_audit(merged, doc)
|
||||
report = _format_audit_report(feature_name, len(merged), audit_results)
|
||||
|
||||
logger.info("[PIPELINE] ✅ Stage 3 完成: %d rules, %d audit issues", len(merged), audit_results.get("issue_count", 0))
|
||||
return {"ir": ir_final, "audit": audit_results, "audit_report": report}
|
||||
|
||||
|
||||
def _generate_audit(rules: list[dict], doc: dict) -> dict:
|
||||
"""Run coverage audits and return structured results."""
|
||||
results = []
|
||||
|
||||
# 1. Source coverage: check that all sections with content are referenced
|
||||
all_sources = set()
|
||||
for rule in rules:
|
||||
for src in rule.get("sources", []):
|
||||
section = src.get("section", "")
|
||||
if section:
|
||||
all_sources.add(section)
|
||||
|
||||
doc_sections = set()
|
||||
for section in doc.get("sections", []):
|
||||
source = section.get("source", "")
|
||||
if source and section.get("blocks"):
|
||||
doc_sections.add(source)
|
||||
|
||||
uncovered = doc_sections - all_sources
|
||||
results.append({
|
||||
"check": "文档章节覆盖",
|
||||
"status": PASS if not uncovered else WARN,
|
||||
"detail": f"已覆盖 {len(all_sources)}/{len(doc_sections)} 章节"
|
||||
+ (f"; 未覆盖: {list(uncovered)[:5]}" if uncovered else ""),
|
||||
})
|
||||
|
||||
# 2. Image coverage
|
||||
img_count = len(doc.get("image_analysis", []))
|
||||
img_referenced = set()
|
||||
for rule in rules:
|
||||
for src in rule.get("sources", []):
|
||||
if src.get("type") in ("image", "logic_tree"):
|
||||
img_referenced.add(src.get("image_id", ""))
|
||||
results.append({
|
||||
"check": "图片覆盖率",
|
||||
"status": PASS if len(img_referenced) >= img_count else (WARN if img_referenced else FAIL),
|
||||
"detail": f"已引用 {len(img_referenced)}/{img_count} 张图片",
|
||||
})
|
||||
|
||||
# 3. Priority distribution
|
||||
p0 = sum(1 for r in rules if r.get("priority") == "P0")
|
||||
p1 = sum(1 for r in rules if r.get("priority") == "P1")
|
||||
p2 = sum(1 for r in rules if r.get("priority") == "P2")
|
||||
results.append({
|
||||
"check": "优先级分布",
|
||||
"status": PASS if p0 > 0 else WARN,
|
||||
"detail": f"P0={p0}, P1={p1}, P2={p2}",
|
||||
})
|
||||
|
||||
issue_count = sum(1 for r in results if r["status"] in (WARN, FAIL))
|
||||
return {"checks": results, "issue_count": issue_count}
|
||||
|
||||
|
||||
def _sync_call(async_fn, *args, **kwargs):
|
||||
"""Run an async function synchronously using the existing event loop if available."""
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# No event loop running — just use asyncio.run
|
||||
return asyncio.run(async_fn(*args, **kwargs))
|
||||
else:
|
||||
# Event loop is running (e.g., inside FastAPI) — must use run_until_complete
|
||||
# Create a new loop in a thread to avoid conflicts
|
||||
import concurrent.futures
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
|
||||
future = ex.submit(asyncio.run, async_fn(*args, **kwargs))
|
||||
return future.result()
|
||||
|
||||
|
||||
def _format_audit_report(feature_name: str, rule_count: int, audit: dict) -> str:
|
||||
"""Generate Markdown audit report."""
|
||||
lines = [
|
||||
f"# IR 完整性审计报告",
|
||||
f"",
|
||||
f"**功能**: {feature_name}",
|
||||
f"**规则总数**: {rule_count}",
|
||||
f"**生成时间**: {datetime.now(timezone.utc).isoformat()}",
|
||||
f"",
|
||||
f"> ⚠️ 请人工审查以下 ⚠️ 和 ❌ 项。",
|
||||
f"",
|
||||
f"## 审计结果",
|
||||
f"",
|
||||
f"| 检查项 | 状态 | 详情 |",
|
||||
f"|--------|------|------|",
|
||||
]
|
||||
for r in audit.get("checks", []):
|
||||
lines.append(f"| {r['check']} | {r['status']} | {r['detail']} |")
|
||||
lines.append("")
|
||||
lines.append(f"**审计问题总计**: {audit.get('issue_count', 0)} 个需要人工审查")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Pipeline Orchestrator
|
||||
# =============================================================================
|
||||
|
||||
class IRPipeline:
|
||||
"""Orchestrates the 3-stage IR generation pipeline."""
|
||||
|
||||
def __init__(self):
|
||||
self._store: dict[str, dict] = {}
|
||||
|
||||
def run_streaming(self, prd_id: str, parsed_doc: dict):
|
||||
"""Run pipeline as a sync generator yielding progress events.
|
||||
|
||||
This is a plain generator (NOT async) so it never blocks the event loop
|
||||
between yields. The SSE endpoint wraps it in a thread + Queue bridge.
|
||||
"""
|
||||
t_total_start = time.time()
|
||||
n_sections = len(parsed_doc.get("sections", []))
|
||||
est_units = max(n_sections * 2, 3)
|
||||
est_total = 20 + (est_units * 20 / 3) + 1
|
||||
|
||||
# --- Stage 1 ---
|
||||
yield {"stage": 1, "status": "running", "message": "Stage 1: 分析文档结构,识别功能单元...",
|
||||
"elapsed": 0, "stage_total": 3, "detail": f"文档含 {n_sections} 个章节"}
|
||||
t0 = time.time()
|
||||
try:
|
||||
semantic_index = _sync_call(stage1_semantic_index, parsed_doc)
|
||||
except Exception as e:
|
||||
yield {"stage": 1, "status": "error", "message": f"Stage 1 失败: {e}", "stage_total": 3}
|
||||
return
|
||||
t1 = time.time()
|
||||
n_units = len(semantic_index.get("function_units", []))
|
||||
est_total = 20 + (max(n_units, 1) * 25 / 3) + 1
|
||||
yield {"stage": 1, "status": "done", "message": f"Stage 1 完成: 识别 {n_units} 个功能单元",
|
||||
"elapsed": round(t1 - t0, 1), "stage_total": 3, "detail": f"识别 {n_units} 个功能单元",
|
||||
"n_units": n_units, "estimated_total": round(est_total)}
|
||||
|
||||
# --- Stage 2 ---
|
||||
est_stage2 = max(n_units, 1) * 25 / min(MAX_CONCURRENCY, max(n_units, 1))
|
||||
yield {"stage": 2, "status": "running", "message": f"Stage 2: 逐单元提取 IR 规则 ({n_units} 个单元)...",
|
||||
"elapsed": 0, "stage_total": 3, "detail": f"预计 {est_stage2:.0f}s,{n_units} 个单元 × {min(MAX_CONCURRENCY, n_units)} 并发",
|
||||
"n_units": n_units, "estimated_total": round(est_total)}
|
||||
t1b = time.time()
|
||||
completed = 0
|
||||
total_rules = 0
|
||||
|
||||
fragments = []
|
||||
function_units = semantic_index.get("function_units", [])
|
||||
|
||||
# Optimization: filter out trivial units (very short desc, no text sources)
|
||||
filtered_units = []
|
||||
for fu in function_units:
|
||||
sources = fu.get("sources", [])
|
||||
desc = fu.get("description", "")
|
||||
has_text = any(s.get("type") in ("para", "table") for s in sources)
|
||||
if len(desc) < 15 and not has_text:
|
||||
logger.info("[PIPELINE] 跳过空单元: %s", fu.get("unit_id", "?"))
|
||||
continue
|
||||
filtered_units.append(fu)
|
||||
|
||||
skipped = len(function_units) - len(filtered_units)
|
||||
if skipped:
|
||||
logger.info("[PIPELINE] Stage 2 过滤掉 %d 个空单元,剩余 %d", skipped, len(filtered_units))
|
||||
function_units = filtered_units
|
||||
|
||||
image_sources = parsed_doc.get("image_sources", {})
|
||||
template_path = PROMPTS_DIR / "ir_step2_extraction.txt"
|
||||
template = template_path.read_text(encoding="utf-8") if template_path.exists() else ""
|
||||
client = router.get_text_client()
|
||||
packages = [build_context_package(fu, parsed_doc, image_sources) for fu in function_units]
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENCY) as executor:
|
||||
futures = {}
|
||||
for i, pkg in enumerate(packages):
|
||||
future = executor.submit(self._extract_one_sync, template, pkg, client)
|
||||
futures[future] = (i, pkg["unit_id"], pkg["unit_name"])
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
i, uid, uname = futures[future]
|
||||
try:
|
||||
rules = future.result()
|
||||
fragments.append({"unit_id": uid, "unit_name": uname, "rules": rules})
|
||||
total_rules += len(rules)
|
||||
except Exception as e:
|
||||
logger.warning("Unit %s failed: %s", uid, e)
|
||||
fragments.append({"unit_id": uid, "unit_name": uname, "rules": [], "error": str(e)})
|
||||
|
||||
completed += 1
|
||||
elapsed = round(time.time() - t1b, 1)
|
||||
yield {"stage": 2, "status": "running",
|
||||
"message": f"Stage 2: {completed}/{n_units} 单元完成 ({total_rules} 条规则)",
|
||||
"elapsed": elapsed, "stage_total": 3,
|
||||
"detail": f"已完成 {completed}/{n_units},累计 {total_rules} 条规则",
|
||||
"completed": completed, "total": n_units, "rules_so_far": total_rules,
|
||||
"estimated_total": round(est_total)}
|
||||
|
||||
fragments.sort(key=lambda f: f["unit_id"])
|
||||
t2 = time.time()
|
||||
logger.info("[PIPELINE] ✅ Stage 2 完成: %d fragments, %d total rules", len(fragments), total_rules)
|
||||
yield {"stage": 2, "status": "done",
|
||||
"message": f"Stage 2 完成: {len(fragments)} 个单元 → {total_rules} 条规则",
|
||||
"elapsed": round(t2 - t1b, 1), "stage_total": 3,
|
||||
"detail": f"{len(fragments)} 个单元 → {total_rules} 条规则",
|
||||
"total_rules": total_rules}
|
||||
|
||||
# --- Stage 3 ---
|
||||
yield {"stage": 3, "status": "running", "message": "Stage 3: 合并去重 + 审计报告...",
|
||||
"elapsed": 0, "stage_total": 3, "detail": "去重 + 分配 ID + 生成审计报告"}
|
||||
result = stage3_merge_and_audit(fragments, semantic_index, parsed_doc)
|
||||
t3 = time.time()
|
||||
n_rules = len(result["ir"]["rules"])
|
||||
|
||||
ir_id = str(uuid.uuid4())[:8]
|
||||
output = {
|
||||
"ir_id": ir_id, "prd_id": prd_id,
|
||||
"yaml_content": json.dumps(result["ir"], ensure_ascii=False, indent=2),
|
||||
"ir_json": result["ir"],
|
||||
"audit": result["audit"],
|
||||
"audit_report": result["audit_report"],
|
||||
"skill_used": "default",
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"pipeline_stats": {
|
||||
"stage1_seconds": round(t1 - t0, 1),
|
||||
"stage2_seconds": round(t2 - t1b, 1),
|
||||
"stage3_seconds": round(t3 - t2, 1),
|
||||
"total_seconds": round(t3 - t_total_start, 1),
|
||||
},
|
||||
}
|
||||
self._store[ir_id] = output
|
||||
|
||||
# Save to disk
|
||||
output_dir = settings.OUTPUT_DIR / prd_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_dir / f"ir_{ir_id}.json", "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
yield {"stage": 3, "status": "done", "message": f"流水线完成!共 {n_rules} 条规则",
|
||||
"elapsed": round(t3 - t2, 1), "stage_total": 3,
|
||||
"detail": f"合并后 {n_rules} 条规则,审计问题 {result['audit'].get('issue_count', 0)} 个",
|
||||
"done": True, "result": output}
|
||||
|
||||
def _extract_one_sync(self, template: str, pkg: dict, client) -> list[dict]:
|
||||
"""Synchronous rule extraction for one unit (runs in thread pool)."""
|
||||
prompt = _build_stage2_prompt(template, pkg)
|
||||
for attempt in range(3):
|
||||
try:
|
||||
raw = client.chat(
|
||||
model=router.text_model,
|
||||
messages=[
|
||||
{"role": "system", "content": "你是一个精确的 JSON 输出引擎。只输出合法的 JSON 数组,不输出任何其他文字。"},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
temperature=0.1,
|
||||
)
|
||||
json_str = extract_json_from_response(raw, expect_array=True)
|
||||
rules = json.loads(json_str)
|
||||
if isinstance(rules, list):
|
||||
return rules
|
||||
except Exception as e:
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(2)
|
||||
return []
|
||||
|
||||
async def run(self, prd_id: str, parsed_doc: dict) -> dict:
|
||||
"""Run the full 3-stage pipeline and return the final IR + audit."""
|
||||
logger.info("[PIPELINE] ═══ 流水线启动 for PRD %s", prd_id)
|
||||
|
||||
# Stage 1
|
||||
t0 = time.time()
|
||||
semantic_index = await stage1_semantic_index(parsed_doc)
|
||||
t1 = time.time()
|
||||
logger.info("[PIPELINE] Stage 1 耗时: %.1fs", t1 - t0)
|
||||
|
||||
# Stage 2
|
||||
fragments = await stage2_extract_rules(semantic_index, parsed_doc)
|
||||
t2 = time.time()
|
||||
logger.info("[PIPELINE] Stage 2 耗时 %.1fs", t2 - t1)
|
||||
|
||||
# Stage 3
|
||||
result = stage3_merge_and_audit(fragments, semantic_index, parsed_doc)
|
||||
t3 = time.time()
|
||||
logger.info("[PIPELINE] Stage 3 耗时 %.1fs", t3 - t2)
|
||||
|
||||
ir_id = str(uuid.uuid4())[:8]
|
||||
output = {
|
||||
"ir_id": ir_id,
|
||||
"prd_id": prd_id,
|
||||
"yaml_content": json.dumps(result["ir"], ensure_ascii=False, indent=2),
|
||||
"ir_json": result["ir"],
|
||||
"audit": result["audit"],
|
||||
"audit_report": result["audit_report"],
|
||||
"skill_used": "default",
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"pipeline_stats": {
|
||||
"stage1_seconds": round(t1 - t0, 1),
|
||||
"stage2_seconds": round(t2 - t1, 1),
|
||||
"stage3_seconds": round(t3 - t2, 1),
|
||||
"total_seconds": round(t3 - t0, 1),
|
||||
},
|
||||
}
|
||||
self._store[ir_id] = output
|
||||
|
||||
# Save output
|
||||
output_dir = settings.OUTPUT_DIR / prd_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_dir / f"ir_{ir_id}.json", "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
logger.info("[PIPELINE] ✅ 流水线完成: ir_id=%s, rules=%d, total=%.1fs",
|
||||
ir_id, len(result["ir"]["rules"]), t3 - t0)
|
||||
return output
|
||||
|
||||
async def get_ir(self, ir_id: str) -> dict | None:
|
||||
return self._store.get(ir_id)
|
||||
|
||||
|
||||
ir_pipeline = IRPipeline()
|
||||
@@ -0,0 +1,100 @@
|
||||
# IR 验证器:Schema 校验 + Principles 规则检查
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
import jsonschema
|
||||
import yaml
|
||||
|
||||
from server.config import settings
|
||||
from server.services.ir_engine.generator import ir_generator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IRValidator:
|
||||
"""Validate IR against skill schema and soul principles."""
|
||||
|
||||
async def validate(self, ir_yaml: str, skill_name: str = "default") -> dict:
|
||||
"""Validate IR YAML content and return {valid, issues}."""
|
||||
issues = []
|
||||
|
||||
# 1. Parse YAML
|
||||
try:
|
||||
ir_data = yaml.safe_load(ir_yaml)
|
||||
except yaml.YAMLError as e:
|
||||
return {"valid": False, "issues": [{
|
||||
"severity": "error",
|
||||
"message": f"YAML 解析失败: {e}",
|
||||
}]}
|
||||
|
||||
if not isinstance(ir_data, dict):
|
||||
return {"valid": False, "issues": [{
|
||||
"severity": "error",
|
||||
"message": "IR 内容必须是 YAML 字典/对象",
|
||||
}]}
|
||||
|
||||
# 2. Schema validation
|
||||
skill = ir_generator.load_skill(skill_name)
|
||||
schema = skill.get("schema", {})
|
||||
if schema:
|
||||
try:
|
||||
jsonschema.validate(instance=ir_data, schema=schema)
|
||||
except jsonschema.ValidationError as e:
|
||||
issues.append({
|
||||
"severity": "error",
|
||||
"message": f"Schema 校验失败: {e.message}",
|
||||
"location": ".".join(str(p) for p in e.absolute_path),
|
||||
})
|
||||
|
||||
# 3. Principles validation
|
||||
principles = self._load_principles()
|
||||
for principle in principles:
|
||||
rule = principle.get("rule", "")
|
||||
pid = principle.get("id", "")
|
||||
|
||||
# Check completeness: each feature must have both positive and negative paths
|
||||
if pid == "completeness":
|
||||
features = ir_data.get("features", [])
|
||||
if not features:
|
||||
issues.append({
|
||||
"severity": "warning",
|
||||
"message": f"原则 '{pid}': 未检测到功能点,无法验证完整性",
|
||||
})
|
||||
|
||||
# Check traceability: features should have descriptions
|
||||
if pid == "traceability":
|
||||
features = ir_data.get("features", [])
|
||||
for i, feat in enumerate(features):
|
||||
if not feat.get("description"):
|
||||
issues.append({
|
||||
"severity": "warning",
|
||||
"message": f"原则 '{pid}': 功能点 {feat.get('feature_name', f'#{i}')} 缺少描述",
|
||||
})
|
||||
|
||||
# Check no-hallucination: warn about potentially vague features
|
||||
if pid == "no-hallucination":
|
||||
features = ir_data.get("features", [])
|
||||
for feat in features:
|
||||
desc = feat.get("description", "")
|
||||
if "应该" in desc or "可能" in desc or "猜测" in desc:
|
||||
issues.append({
|
||||
"severity": "warning",
|
||||
"message": f"原则 '{pid}': 功能点 '{feat.get('feature_name', '')}' 描述包含不确定措辞",
|
||||
})
|
||||
|
||||
valid = len([i for i in issues if i["severity"] == "error"]) == 0
|
||||
logger.info("IR validation: valid=%s, issues=%d", valid, len(issues))
|
||||
return {"valid": valid, "issues": issues}
|
||||
|
||||
def _load_principles(self) -> list[dict]:
|
||||
"""Load principles from .zeekerwatchmen/soul/principles.yaml."""
|
||||
principles_path = settings.SOUL_DIR / "principles.yaml"
|
||||
if not principles_path.exists():
|
||||
return []
|
||||
with open(principles_path, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
return data.get("principles", []) if data else []
|
||||
|
||||
|
||||
ir_validator = IRValidator()
|
||||
@@ -0,0 +1,85 @@
|
||||
# Vision LLM wrapper for analyzing document images (type + description).
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from server.core.llm_provider.router import router
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROMPT_IMAGE = """请分析这张图片,判断类型并输出文字描述。
|
||||
|
||||
## 判断图片类型
|
||||
|
||||
如果是 **流程图 / 架构图 / 状态图 / 时序图 / 活动图**,详细描述:
|
||||
- 图中所有节点/步骤/状态/组件的名称
|
||||
- 所有连线/箭头/转换关系及其方向
|
||||
- 所有分支条件、判断逻辑和判断结果
|
||||
- 所有文字标注、注释、标签
|
||||
- 图的整体结构和逻辑流程
|
||||
- 如果图片包含多个子图,拆解描述
|
||||
|
||||
如果是 **其他类型**(UI原型图 / 界面截图 / 设计稿 / 手机屏幕截图 / 网页截图等),简要描述图片内容。
|
||||
|
||||
## 输出格式
|
||||
|
||||
**1. 类型标签(单独一行):**
|
||||
type: <flowchart|architecture|state|sequence|activity|other>
|
||||
|
||||
**2. 文字描述:**
|
||||
该图片的详细文字描述。
|
||||
|
||||
不要输出 ---YAML--- 分隔符或 YAML 内容,不要添加任何额外的解释或问候语。"""
|
||||
|
||||
|
||||
class ImageParser:
|
||||
"""Analyze document images using Qwen VL model.
|
||||
|
||||
Usage::
|
||||
|
||||
parser = ImageParser()
|
||||
result = parser.parse_image("images/flow.png")
|
||||
# {"type": "flowchart", "description": "..."}
|
||||
"""
|
||||
|
||||
_VALID_TYPES = {"flowchart", "architecture", "state", "sequence", "activity", "other"}
|
||||
|
||||
def __init__(self):
|
||||
self._llm = router.get_image_client()
|
||||
|
||||
@property
|
||||
def usage(self) -> dict:
|
||||
return self._llm.usage
|
||||
|
||||
def parse_image(self, image_path: str) -> Optional[dict]:
|
||||
"""Parse an image and return {type, description}."""
|
||||
logger.info("Parsing image: %s", image_path)
|
||||
|
||||
try:
|
||||
content = self._llm.chat_with_image(
|
||||
model=router.image_model,
|
||||
image_path=image_path,
|
||||
prompt=PROMPT_IMAGE,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Image parsing failed: %s", e)
|
||||
return {"type": "other", "description": "", "error": str(e)}
|
||||
|
||||
return self._parse_response(content)
|
||||
|
||||
def _parse_response(self, content: str) -> dict:
|
||||
"""Extract (type, description) from vision model response."""
|
||||
content = content.strip()
|
||||
parsed_type = "other"
|
||||
desc_lines: list[str] = []
|
||||
|
||||
for line in content.splitlines():
|
||||
stripped = line.strip()
|
||||
if (stripped.startswith("type:") or stripped.startswith("类型:")):
|
||||
type_val = stripped.split(":", 1)[1].strip().lower()
|
||||
if type_val in self._VALID_TYPES:
|
||||
parsed_type = type_val
|
||||
else:
|
||||
desc_lines.append(line)
|
||||
|
||||
return {"type": parsed_type, "description": "\n".join(desc_lines).strip()}
|
||||
@@ -0,0 +1,260 @@
|
||||
# PRD 管理服务:上传、解析、版本快照
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from server.config import settings
|
||||
from server.services.prd_manager.word_parser import WordParser
|
||||
from server.services.prd_manager.image_parser import ImageParser
|
||||
|
||||
logger = logging.getLogger("testflow")
|
||||
RATE_LIMIT_DELAY = 0.5
|
||||
|
||||
|
||||
class PRDService:
|
||||
"""PRD management: handle file upload, text extraction, and version snapshots."""
|
||||
|
||||
def __init__(self):
|
||||
self.output_dir = settings.OUTPUT_DIR
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
self._prds: dict[str, dict] = {}
|
||||
|
||||
async def upload_and_parse(self, filename: str, content: bytes) -> dict:
|
||||
prd_id = str(uuid.uuid4())[:8]
|
||||
basename = Path(filename).stem
|
||||
ext = Path(filename).suffix.lower()
|
||||
|
||||
upload_dir = self.output_dir / prd_id
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
filepath = upload_dir / filename
|
||||
|
||||
t0 = time.time()
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
logger.info("[UPLOAD] 收到文件: %s (%.1f KB, %s)", filename, len(content) / 1024, ext)
|
||||
|
||||
if ext in (".txt", ".md"):
|
||||
text = content.decode("utf-8", errors="replace")
|
||||
sections = self._parse_markdown_sections(text)
|
||||
parsed = {
|
||||
"source": str(filepath), "sections": sections,
|
||||
"image_sources": {}, "image_analysis": [], "full_text": text,
|
||||
}
|
||||
logger.info("[PARSE] %.4s %s → %d 章节, %d 字符, %.1fs",
|
||||
prd_id, ext, len(sections), len(text), time.time() - t0)
|
||||
|
||||
elif ext == ".docx":
|
||||
logger.info("[PARSE] %.4s → 开始解析 .docx...", prd_id)
|
||||
parsed = await self._parse_docx(str(filepath), upload_dir)
|
||||
logger.info("[PARSE] %.4s .docx 完成: %d 章节, %.1fs",
|
||||
prd_id, len(parsed.get("sections", [])), time.time() - t0)
|
||||
|
||||
elif ext == ".pdf":
|
||||
logger.info("[PARSE] %.4s → 开始解析 .pdf...", prd_id)
|
||||
parsed = await self._parse_pdf(str(filepath), upload_dir)
|
||||
logger.info("[PARSE] %.4s .pdf 完成: %d 页, %.1fs",
|
||||
prd_id, len(parsed.get("sections", [])), time.time() - t0)
|
||||
else:
|
||||
raise ValueError(f"Unsupported file format: {ext}")
|
||||
|
||||
parsed_path = upload_dir / f"{basename}_parsed.json"
|
||||
with open(parsed_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed, f, ensure_ascii=False, indent=2)
|
||||
|
||||
prd_version = {
|
||||
"prd_id": prd_id, "filename": filename,
|
||||
"uploaded_at": datetime.now(timezone.utc).isoformat(), "status": "ready",
|
||||
"parsed_path": str(parsed_path), "full_text": parsed.get("full_text", ""),
|
||||
"sections": parsed.get("sections", []), "images": parsed.get("image_analysis", []),
|
||||
}
|
||||
self._prds[prd_id] = prd_version
|
||||
|
||||
n_paras = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "para")
|
||||
n_tables = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "table")
|
||||
logger.info("[UPLOAD] %.4s 解析完成: %d章节 %d段落 %d表格 %d图片 → %s",
|
||||
prd_id, len(parsed.get("sections", [])), n_paras, n_tables,
|
||||
len(parsed.get("image_analysis", [])), parsed_path)
|
||||
return prd_version
|
||||
|
||||
async def upload_quick(self, filename: str, content: bytes) -> dict:
|
||||
"""Quick save — returns immediately, parsing happens in background."""
|
||||
prd_id = str(uuid.uuid4())[:8]
|
||||
upload_dir = self.output_dir / prd_id
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
filepath = upload_dir / filename
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
logger.info("[UPLOAD] %.4s 快速保存: %s (%.1f KB)", prd_id, filename, len(content) / 1024)
|
||||
|
||||
prd_version = {
|
||||
"prd_id": prd_id, "filename": filename,
|
||||
"uploaded_at": datetime.now(timezone.utc).isoformat(),
|
||||
"status": "parsing", "full_text": "", "sections": [], "images": [],
|
||||
"parsed_path": "", "filepath": str(filepath),
|
||||
}
|
||||
self._prds[prd_id] = prd_version
|
||||
return prd_version
|
||||
|
||||
async def parse_async(self, prd_id: str, filename: str, content: bytes):
|
||||
"""Full parse (text + images + vision) — called in background thread."""
|
||||
basename = Path(filename).stem
|
||||
ext = Path(filename).suffix.lower()
|
||||
upload_dir = self.output_dir / prd_id
|
||||
filepath = upload_dir / filename
|
||||
t0 = time.time()
|
||||
|
||||
# Parse
|
||||
if ext in (".txt", ".md"):
|
||||
text = content.decode("utf-8", errors="replace")
|
||||
sections = self._parse_markdown_sections(text)
|
||||
parsed = {"source": str(filepath), "sections": sections,
|
||||
"image_sources": {}, "image_analysis": [], "full_text": text}
|
||||
elif ext == ".docx":
|
||||
parsed = await self._parse_docx(str(filepath), upload_dir)
|
||||
elif ext == ".pdf":
|
||||
parsed = await self._parse_pdf(str(filepath), upload_dir)
|
||||
else:
|
||||
self.set_status(prd_id, "error", f"Unsupported format: {ext}")
|
||||
return
|
||||
|
||||
# Save parsed
|
||||
parsed_path = upload_dir / f"{basename}_parsed.json"
|
||||
with open(parsed_path, "w", encoding="utf-8") as f:
|
||||
json.dump(parsed, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# Update in-memory
|
||||
prd = self._prds.get(prd_id, {})
|
||||
prd.update({
|
||||
"status": "ready", "parsed_path": str(parsed_path),
|
||||
"full_text": parsed.get("full_text", ""),
|
||||
"sections": parsed.get("sections", []),
|
||||
"images": parsed.get("image_analysis", []),
|
||||
})
|
||||
|
||||
n_paras = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "para")
|
||||
n_tables = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "table")
|
||||
logger.info("[UPLOAD] %.4s 后台解析完成: %d章节 %d段落 %d表格 %d图片, %.1fs",
|
||||
prd_id, len(parsed.get("sections", [])), n_paras, n_tables,
|
||||
len(parsed.get("image_analysis", [])), time.time() - t0)
|
||||
|
||||
def set_status(self, prd_id: str, status: str, error: str = ""):
|
||||
if prd_id in self._prds:
|
||||
self._prds[prd_id]["status"] = status
|
||||
if error:
|
||||
self._prds[prd_id]["error"] = error
|
||||
|
||||
async def get_status(self, prd_id: str) -> dict | None:
|
||||
prd = self._prds.get(prd_id)
|
||||
if not prd:
|
||||
return None
|
||||
return {"prd_id": prd_id, "status": prd.get("status", "ready"),
|
||||
"error": prd.get("error", ""),
|
||||
"sections_count": len(prd.get("sections", [])),
|
||||
"images_count": len(prd.get("images", []))}
|
||||
|
||||
async def get_prd(self, prd_id: str) -> dict | None:
|
||||
return self._prds.get(prd_id)
|
||||
|
||||
# ---- internal parsers ----
|
||||
|
||||
async def _parse_docx(self, filepath: str, output_dir: Path) -> dict:
|
||||
word = WordParser(filepath)
|
||||
sections, image_sources = word.extract_sections()
|
||||
full_text = word.extract_full_text()
|
||||
|
||||
images_dir = output_dir / "images"
|
||||
images = word.extract_images(str(images_dir))
|
||||
image_analysis = []
|
||||
n_chart_images = 0
|
||||
|
||||
if images:
|
||||
parser = ImageParser()
|
||||
logger.info("[IMAGE] %.4s 发现 %d 张图片,开始视觉分析...", output_dir.parent.name, len(images))
|
||||
for i, img in enumerate(images):
|
||||
t_img = time.time()
|
||||
logger.info("[IMAGE] [%d/%d] rid=%s, 调用 Qwen VL...", i + 1, len(images), img["rid"])
|
||||
result = parser.parse_image(img["path"])
|
||||
if result is None:
|
||||
result = {"type": "other", "description": ""}
|
||||
result["rid"] = img["rid"]
|
||||
result["path"] = img["path"]
|
||||
if img["rid"] in image_sources:
|
||||
result["context"] = image_sources[img["rid"]]
|
||||
image_analysis.append(result)
|
||||
if result.get("type") in ("flowchart", "architecture", "state", "sequence", "activity"):
|
||||
n_chart_images += 1
|
||||
logger.info("[IMAGE] [%d/%d] rid=%s 完成,类型=%s (%.1fs)",
|
||||
i + 1, len(images), img["rid"], result.get("type", "?"), time.time() - t_img)
|
||||
if i < len(images) - 1:
|
||||
time.sleep(RATE_LIMIT_DELAY)
|
||||
|
||||
usg = parser.usage
|
||||
logger.info("[IMAGE] %.4s 图片分析完成: %d张, 图表类%d张, tokens(p=%d c=%d t=%d)",
|
||||
output_dir.parent.name, len(images), n_chart_images,
|
||||
usg["prompt_tokens"], usg["completion_tokens"], usg["total_tokens"])
|
||||
else:
|
||||
logger.info("[IMAGE] %.4s 文档无图片", output_dir.parent.name)
|
||||
|
||||
return {
|
||||
"source": filepath, "sections": sections,
|
||||
"image_sources": image_sources, "image_analysis": image_analysis,
|
||||
"full_text": full_text,
|
||||
}
|
||||
|
||||
def _parse_markdown_sections(self, text: str) -> list[dict]:
|
||||
import re
|
||||
sections: list[dict] = []
|
||||
current_source = "正文"
|
||||
current_blocks: list[dict] = []
|
||||
para_idx = 0
|
||||
|
||||
for line in text.split("\n"):
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
heading_match = re.match(r"^(#{1,6})\s+(.+)", stripped)
|
||||
if heading_match:
|
||||
if current_blocks:
|
||||
sections.append({"source": current_source, "blocks": current_blocks, "images": []})
|
||||
current_blocks = []
|
||||
level = len(heading_match.group(1))
|
||||
current_source = f"{'#' * level} {heading_match.group(2)}"
|
||||
continue
|
||||
if "|" in stripped and stripped.count("|") >= 2:
|
||||
cells = [c.strip() for c in stripped.split("|") if c.strip()]
|
||||
if cells:
|
||||
current_blocks.append({
|
||||
"type": "table",
|
||||
"table": len([b for b in current_blocks if b["type"] == "table"]) + 1,
|
||||
"headers": cells,
|
||||
"rows": [{"columns": [{"name": c, "row": 1, "col": i + 1, "text": c} for i, c in enumerate(cells)]}],
|
||||
})
|
||||
continue
|
||||
para_idx += 1
|
||||
current_blocks.append({"type": "para", "index": para_idx, "text": stripped})
|
||||
|
||||
if current_blocks:
|
||||
sections.append({"source": current_source, "blocks": current_blocks, "images": []})
|
||||
return sections
|
||||
|
||||
async def _parse_pdf(self, filepath: str, output_dir: Path) -> dict:
|
||||
from PyPDF2 import PdfReader
|
||||
reader = PdfReader(filepath)
|
||||
full_text_parts = []
|
||||
for i, page in enumerate(reader.pages):
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
full_text_parts.append(text)
|
||||
full_text = "\n".join(full_text_parts)
|
||||
sections = [{"source": "PDF 内容", "blocks": [{"type": "para", "index": i + 1, "text": t} for i, t in enumerate(full_text_parts)], "images": []}]
|
||||
return {"source": filepath, "sections": sections, "image_sources": {}, "image_analysis": [], "full_text": full_text}
|
||||
|
||||
|
||||
prd_service = PRDService()
|
||||
@@ -0,0 +1,255 @@
|
||||
# Microsoft Word (.docx) parser — extracts structured sections with typed blocks,
|
||||
# image markers, and detailed location metadata for downstream IR generation.
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from docx import Document
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_EXT = {
|
||||
"image/png": ".png", "image/jpeg": ".jpg", "image/gif": ".gif",
|
||||
"image/bmp": ".bmp", "image/tiff": ".tiff", "image/webp": ".webp",
|
||||
"image/x-emf": ".emf", "image/x-wmf": ".wmf", "image/svg+xml": ".svg",
|
||||
}
|
||||
|
||||
HEADER_CELL_MAX_LEN = 20
|
||||
|
||||
WML_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
||||
DRAW_NS = "http://schemas.openxmlformats.org/drawingml/2006/main"
|
||||
REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
||||
|
||||
|
||||
class WordParser:
|
||||
"""Parse a .docx file — extract images, split body into sections.
|
||||
|
||||
Sections contain typed blocks (para/table) with structured table columns
|
||||
and IMAGE:rid markers, matching the reference document analyzer format.
|
||||
|
||||
Usage::
|
||||
|
||||
parser = WordParser("doc.docx")
|
||||
parser.extract_images("images/")
|
||||
sections, image_sources = parser.extract_sections()
|
||||
"""
|
||||
|
||||
def __init__(self, docx_path: str):
|
||||
if not os.path.isfile(docx_path):
|
||||
raise FileNotFoundError(f"Document not found: {docx_path}")
|
||||
self._doc = Document(docx_path)
|
||||
|
||||
# ---- public API ---------------------------------------------------------
|
||||
|
||||
def extract_images(self, images_dir: str) -> list[dict]:
|
||||
"""Save all images to *images_dir*. Returns [{rid, path}, ...]."""
|
||||
os.makedirs(images_dir, exist_ok=True)
|
||||
images: list[dict] = []
|
||||
for rel in self._doc.part.rels.values():
|
||||
if "image" not in rel.reltype:
|
||||
continue
|
||||
ext = IMAGE_EXT.get(rel.target_part.content_type, ".png")
|
||||
name = f"image_{rel.rId}{ext}"
|
||||
path = os.path.join(images_dir, name)
|
||||
with open(path, "wb") as f:
|
||||
f.write(rel.target_part.blob)
|
||||
images.append({"rid": rel.rId, "path": path})
|
||||
return images
|
||||
|
||||
def extract_sections(self) -> tuple[list[dict], dict[str, dict]]:
|
||||
"""Walk document body and split into sections by heading.
|
||||
|
||||
Returns:
|
||||
*sections* — [{source, blocks, images}, ...]
|
||||
Each block is {type, index, text} (paragraph) or
|
||||
{type, table, headers, rows} (table with structured columns).
|
||||
*image_sources* — rid → {section, table?, row?, column?, name?}
|
||||
"""
|
||||
sections: list[dict] = []
|
||||
current_source = ""
|
||||
blocks: list[dict] = []
|
||||
section_images: list[str] = []
|
||||
image_sources: dict[str, dict] = {}
|
||||
para_idx = 0
|
||||
tbl_idx = 0
|
||||
|
||||
for child in self._doc.element.body:
|
||||
tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
||||
|
||||
if tag == "p":
|
||||
para = Paragraph(child, self._doc)
|
||||
|
||||
# Heading detection
|
||||
if self._heading_level(para) is not None:
|
||||
heading_text = para.text.strip()
|
||||
if heading_text:
|
||||
# Flush previous section
|
||||
if blocks or section_images:
|
||||
sections.append({
|
||||
"source": current_source,
|
||||
"blocks": blocks,
|
||||
"images": list(section_images),
|
||||
})
|
||||
blocks = []
|
||||
section_images = []
|
||||
para_idx = 0
|
||||
tbl_idx = 0
|
||||
current_source = heading_text
|
||||
continue
|
||||
|
||||
text = para.text.strip()
|
||||
|
||||
# Scan for inline images — append [[IMAGE:rid]] markers
|
||||
for run in para.runs:
|
||||
for rid in self._images_in(run._element):
|
||||
text += f" [[IMAGE:{rid}]]"
|
||||
section_images.append(rid)
|
||||
image_sources[rid] = {"section": current_source}
|
||||
|
||||
if text.strip():
|
||||
blocks.append({"type": "para", "index": para_idx + 1, "text": text.strip()})
|
||||
para_idx += 1
|
||||
|
||||
elif tag == "tbl":
|
||||
tbl_idx += 1
|
||||
table = Table(child, self._doc)
|
||||
|
||||
# Collect all rows with cell text and embedded images
|
||||
all_rows: list[list[str]] = []
|
||||
all_images: list[list[list[str]]] = [] # row → col → [rids]
|
||||
for row in table.rows:
|
||||
row_texts: list[str] = []
|
||||
row_cell_images: list[list[str]] = []
|
||||
for cell in row.cells:
|
||||
cell_text = cell.text.strip()
|
||||
cell_imgs: list[str] = []
|
||||
for cp in cell.paragraphs:
|
||||
for run in cp.runs:
|
||||
for rid in self._images_in(run._element):
|
||||
cell_imgs.append(rid)
|
||||
for rid in cell_imgs:
|
||||
cell_text += f" [[IMAGE:{rid}]]"
|
||||
section_images.append(rid)
|
||||
row_texts.append(cell_text.strip())
|
||||
row_cell_images.append(cell_imgs)
|
||||
if any(row_texts) or any(row_cell_images):
|
||||
all_rows.append(row_texts)
|
||||
all_images.append(row_cell_images)
|
||||
|
||||
if len(all_rows) >= 2:
|
||||
# Header heuristic: first row is header if all cells are short
|
||||
first_row = all_rows[0]
|
||||
has_header = all(len(c) < HEADER_CELL_MAX_LEN for c in first_row)
|
||||
if has_header:
|
||||
headers = first_row
|
||||
data_rows_slice = zip(all_rows[1:], all_images[1:])
|
||||
else:
|
||||
headers = [f"列{ci + 1}" for ci in range(len(first_row))]
|
||||
data_rows_slice = zip(all_rows, all_images)
|
||||
|
||||
data_rows: list[dict] = []
|
||||
for ri, (row_data, row_imgs) in enumerate(data_rows_slice):
|
||||
columns: list[dict] = []
|
||||
max_cols = max(len(headers), len(row_data))
|
||||
for ci in range(max_cols):
|
||||
hdr = headers[ci] if ci < len(headers) else ""
|
||||
txt = row_data[ci] if ci < len(row_data) else ""
|
||||
columns.append({
|
||||
"name": hdr,
|
||||
"row": ri + 1,
|
||||
"col": ci + 1,
|
||||
"text": txt,
|
||||
})
|
||||
|
||||
# Register image sources with structured location
|
||||
imgs = row_imgs[ci] if ci < len(row_imgs) else []
|
||||
for rid in imgs:
|
||||
image_sources[rid] = {
|
||||
"section": current_source,
|
||||
"table": tbl_idx,
|
||||
"row": ri + 1,
|
||||
"column": ci + 1,
|
||||
"name": hdr,
|
||||
}
|
||||
|
||||
data_rows.append({"columns": columns})
|
||||
|
||||
blocks.append({
|
||||
"type": "table",
|
||||
"table": tbl_idx,
|
||||
"headers": headers,
|
||||
"rows": data_rows,
|
||||
})
|
||||
elif all_rows:
|
||||
# Degenerate table — treat as plain rows
|
||||
for ri, row_data in enumerate(all_rows):
|
||||
row_text = " | ".join(row_data)
|
||||
if row_text.strip():
|
||||
blocks.append({
|
||||
"type": "para",
|
||||
"index": para_idx + 1,
|
||||
"text": row_text,
|
||||
})
|
||||
para_idx += 1
|
||||
|
||||
# Flush final section
|
||||
if blocks or section_images:
|
||||
sections.append({
|
||||
"source": current_source,
|
||||
"blocks": blocks,
|
||||
"images": list(section_images),
|
||||
})
|
||||
|
||||
return sections, image_sources
|
||||
|
||||
def extract_full_text(self) -> str:
|
||||
"""Return all text content as a single string."""
|
||||
lines = []
|
||||
for para in self._doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
if self._heading_level(para) is not None:
|
||||
lines.append(f"\n## {text}\n")
|
||||
else:
|
||||
lines.append(text)
|
||||
for table in self._doc.tables:
|
||||
for row in table.rows:
|
||||
lines.append(" | ".join(cell.text.strip() for cell in row.cells))
|
||||
return "\n".join(lines)
|
||||
|
||||
# ---- internals ----------------------------------------------------------
|
||||
|
||||
def _heading_level(self, para: Paragraph) -> int | None:
|
||||
"""Heading level 1-9, or None if not a heading."""
|
||||
if para.style and para.style.name:
|
||||
name = para.style.name
|
||||
for prefix in ("Heading", "标题"):
|
||||
if name.startswith(prefix):
|
||||
try:
|
||||
return int(name.split()[-1])
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
pPr = para._element.find(f"{{{WML_NS}}}pPr")
|
||||
if pPr is not None:
|
||||
ol = pPr.find(f"{{{WML_NS}}}outlineLvl")
|
||||
if ol is not None:
|
||||
val = ol.get(f"{{{WML_NS}}}val")
|
||||
if val is not None:
|
||||
try:
|
||||
return int(val) + 1
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _images_in(self, element) -> list[str]:
|
||||
"""Return rId values for drawings embedded in *element*."""
|
||||
rids: list[str] = []
|
||||
for drawing in element.findall(f".//{{{WML_NS}}}drawing"):
|
||||
blip = drawing.find(f".//{{{DRAW_NS}}}blip")
|
||||
if blip is not None:
|
||||
rid = blip.get(f"{{{REL_NS}}}embed")
|
||||
if rid:
|
||||
rids.append(rid)
|
||||
return rids
|
||||
@@ -0,0 +1,30 @@
|
||||
{persona}
|
||||
|
||||
{principles}
|
||||
|
||||
## 你的能力
|
||||
你运行在 ZeekerWatchman 平台中,是一个测试用例管理助手。你可以帮助用户:
|
||||
1. 分析 PRD 文档,提取功能点
|
||||
2. 审查和修改 IR(中间表示)
|
||||
3. 生成和优化测试用例
|
||||
4. 回答关于测试设计的问题
|
||||
|
||||
## 当前左侧面板状态
|
||||
{page_context}
|
||||
|
||||
## 左侧数据
|
||||
{data_context}
|
||||
|
||||
## 操作指令
|
||||
{action_format}
|
||||
|
||||
### IR 规则 ID 格式
|
||||
IR 规则 ID 格式为 `IR-{FEATURE}-{section}-{NNN}`,如 `IR-ZWM-4.2.1-001`。
|
||||
测试用例 ID 格式为 `TC-{FEATURE}-{NNN}`,如 `TC-ZWM-001`。
|
||||
|
||||
### 操作格式
|
||||
当需要让系统执行操作时,在回复末尾附加 action JSON 块:
|
||||
```action
|
||||
{{"action": "<action_name>", ...}}
|
||||
```
|
||||
每次最多 3 个操作。无操作时不输出。
|
||||
@@ -0,0 +1,56 @@
|
||||
你是一个产品需求分析师。你的任务是从 PRD 文档中提取"语义索引"——一份结构化的功能清单,而不是逐字翻译。
|
||||
|
||||
## 文档结构说明
|
||||
|
||||
下面是一份文档的解析结果,包含:
|
||||
|
||||
1. **sections**:按章节组织的混合内容(段落 + 表格),每个 section 有 `source`(章节标题)、`blocks`(`para` 文本段落和 `table` 结构表格)、`images`(引用的图片 ID 列表)
|
||||
2. **image_analysis**:文档中图片的程序化分析结果,包含图片类型(flowchart/architecture/state/sequence/activity/other)和文字描述
|
||||
3. **resolved_conflicts**:文档中图文冲突的仲裁结果(如有)
|
||||
|
||||
## 文档全文
|
||||
|
||||
{document_json}
|
||||
|
||||
## 你的任务
|
||||
|
||||
阅读整份文档后,输出一份 **语义索引 JSON**,包含:
|
||||
|
||||
### 1. feature_name
|
||||
从文档中识别的主要功能名称
|
||||
|
||||
### 2. concepts
|
||||
文档中定义或使用的关键概念列表。每个概念包含:
|
||||
- `name`:概念的标准名称
|
||||
- `aliases`:同义词/别名列表
|
||||
- `defined_in`:定义该概念的章节号列表
|
||||
|
||||
### 3. function_units
|
||||
文档中描述的所有主要功能行为的列表。**每个 function_unit 对应一条完整的"如果...则..."规则**。每个 function unit 包含:
|
||||
|
||||
- `unit_id`:唯一标识,格式 "FU-001", "FU-002"...
|
||||
- `name`:简短名称
|
||||
- `description`:1-3 句描述该规则的行为
|
||||
- `sources`:该规则在文档中的来源锚点列表,每项包含:
|
||||
- `section`:章节号
|
||||
- `type`:来源类型,"table"/"para"/"logic_tree"/"image"
|
||||
- `row`:如果是表格行(从 1 开始)
|
||||
- `text_snippet`:前 200 字的关键文字
|
||||
- `image_id`:如果是图片来源,填写图片 rId
|
||||
|
||||
## 关键要求
|
||||
|
||||
1. **必须覆盖所有图片分析中的逻辑**:图片描述中的每个行为都要出现在某个 function_unit 中
|
||||
2. **必须覆盖表格中的所有规则**:表格中列出的每条规则都要有对应的 function_unit
|
||||
3. **区分不同的应用场景和状态**:不同前置条件/状态下的行为分别建模
|
||||
4. **包含开关/状态切换**:功能开关"开启"和"关闭"两种状态下的行为都要覆盖
|
||||
|
||||
## 输出格式
|
||||
|
||||
**只输出 JSON,不要有 markdown 代码块标记或其他文字**:
|
||||
|
||||
{
|
||||
"feature_name": "...",
|
||||
"concepts": [ ... ],
|
||||
"function_units": [ ... ]
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
你是一个需求分析专家。你的任务是基于给定的精准上下文包,为单个功能单元(Function Unit)提取详细的 **IR 规则(Intermediate Representation Rule)**。
|
||||
|
||||
## 上下文
|
||||
|
||||
下面是一个功能单元的精准上下文包,包含了从原始需求文档中提取的相关文字、表格和逻辑:
|
||||
|
||||
### 功能单元概要
|
||||
- **unit_id**: {unit_id}
|
||||
- **unit_name**: {unit_name}
|
||||
- **unit_description**: {unit_description}
|
||||
|
||||
### 相关文字段落
|
||||
{texts}
|
||||
|
||||
### 相关表格
|
||||
{tables}
|
||||
|
||||
### 相关图片分析
|
||||
{images}
|
||||
|
||||
### 冲突仲裁(如有)
|
||||
{resolved_conflicts}
|
||||
|
||||
## IR Schema
|
||||
|
||||
你需要为这个功能单元输出一个 **规则数组(rules)**。每条规则遵循以下 schema:
|
||||
|
||||
```json
|
||||
{{
|
||||
"description": "用完整的中文自然语言描述该规则的触发条件和行为,一句话概括",
|
||||
"priority": "P0",
|
||||
"sources": [
|
||||
{{"type": "table", "section": "...", "row": 2, "text_snippet": "..."}},
|
||||
{{"type": "image", "image_id": "rId16", "description": "..."}}
|
||||
],
|
||||
"precondition": {{
|
||||
"app_type": "...",
|
||||
"app_state": "..."
|
||||
}},
|
||||
"trigger": {{
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{{"signal": "车速", "operator": ">=", "value": 15, "unit": "km/h"}},
|
||||
{{"signal": "持续时间", "operator": ">", "value": 5, "unit": "秒"}}
|
||||
]
|
||||
}},
|
||||
"actions": [
|
||||
{{"type": "system", "description": "系统执行的自动动作"}},
|
||||
{{"type": "user_interaction", "description": "对用户的提示", "content": "提示文案内容"}}
|
||||
]
|
||||
}}
|
||||
```
|
||||
|
||||
### 字段说明
|
||||
|
||||
1. **description**: 完整但简洁地描述整个规则,包含前置条件、触发条件和所有动作
|
||||
2. **priority**: P0(核心规则)、P1(重要规则)、P2(边界情况)
|
||||
3. **sources**: 每条规则必须列出所有数据来源,包括引用的表格行和图片
|
||||
4. **precondition**: 规则生效的前置状态条件。可以是空对象 `{{}}`
|
||||
5. **trigger**: 触发条件对象:
|
||||
- `operator`: "AND" 或 "OR"
|
||||
- `conditions`: 条件数组,每个条件有 `signal`、`operator`、`value`,可选 `unit`
|
||||
6. **actions**: 每个动作有 `type`("system"/"user_interaction")和 `description`。用户可见交互包含 `content` 字段
|
||||
|
||||
## 关键要求
|
||||
|
||||
1. **信号和数值必须精确**:不写"车速超过阈值",写 `"车速 >= 15 km/h"`
|
||||
2. **条件必须完整**:文档中的所有触发条件都要出现在 trigger.conditions 中
|
||||
3. **动作类型区分**:系统行为用 "system",用户可见交互用 "user_interaction"
|
||||
4. **多条规则**:如果一个功能单元包含多个独立的行为分支,输出多条规则分别描述
|
||||
|
||||
## 输出格式
|
||||
|
||||
**只输出 JSON 数组,不要有任何其他文字或 markdown 标记**:
|
||||
|
||||
[
|
||||
{{ ... }},
|
||||
{{ ... }}
|
||||
]
|
||||
|
||||
即使只有一个规则,也必须用数组格式 `[...]`。
|
||||
@@ -0,0 +1,108 @@
|
||||
# Skill 管理服务:扫描、验证、列出所有可用 Skill
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from server.config import settings
|
||||
|
||||
logger = logging.getLogger("testflow")
|
||||
|
||||
REQUIRED_SKILL_FILES = ["skill.yaml", "ir_schema.json", "extract_ir_prompt.j2", "gen_cases_prompt.j2"]
|
||||
|
||||
|
||||
class SkillManager:
|
||||
"""Discover and validate skill packages from .zeekerwatchmen/skills/."""
|
||||
|
||||
def __init__(self):
|
||||
self._skills_dir = settings.SKILLS_DIR
|
||||
self._cache: dict[str, dict] = {}
|
||||
self._scan()
|
||||
|
||||
def _scan(self):
|
||||
"""Scan the skills directory and cache valid skills."""
|
||||
self._cache = {}
|
||||
if not self._skills_dir.exists():
|
||||
logger.warning("[SKILL] s directory not found: %s", self._skills_dir)
|
||||
return
|
||||
|
||||
for skill_dir in sorted(self._skills_dir.iterdir()):
|
||||
if not skill_dir.is_dir():
|
||||
continue
|
||||
|
||||
skill_name = skill_dir.name
|
||||
skill = self._load_skill(skill_name)
|
||||
if skill:
|
||||
self._cache[skill_name] = skill
|
||||
logger.info("[SKILL] 加载: %s (v%s)", skill_name, skill.get("version", "?"))
|
||||
|
||||
logger.info("[SKILL] 发现 %d skills: %s", len(self._cache), list(self._cache.keys()))
|
||||
|
||||
def _load_skill(self, skill_name: str) -> dict | None:
|
||||
"""Load and validate a single skill package."""
|
||||
skill_dir = self._skills_dir / skill_name
|
||||
|
||||
# Check required files
|
||||
missing = []
|
||||
for filename in REQUIRED_SKILL_FILES:
|
||||
if not (skill_dir / filename).exists():
|
||||
missing.append(filename)
|
||||
|
||||
if missing:
|
||||
logger.warning("[SKILL] '%s' missing files: %s", skill_name, missing)
|
||||
return None
|
||||
|
||||
# Load metadata
|
||||
try:
|
||||
with open(skill_dir / "skill.yaml", "r", encoding="utf-8") as f:
|
||||
meta = yaml.safe_load(f) or {}
|
||||
except yaml.YAMLError as e:
|
||||
logger.error("[SKILL] '%s': invalid skill.yaml: %s", skill_name, e)
|
||||
return None
|
||||
|
||||
# Load schema
|
||||
try:
|
||||
with open(skill_dir / "ir_schema.json", "r", encoding="utf-8") as f:
|
||||
schema = json.load(f)
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
logger.error("[SKILL] '%s': invalid ir_schema.json: %s", skill_name, e)
|
||||
return None
|
||||
|
||||
return {
|
||||
"name": skill_name,
|
||||
"display_name": meta.get("display_name", skill_name),
|
||||
"description": meta.get("description", ""),
|
||||
"version": meta.get("version", "0.0.0"),
|
||||
"domain": meta.get("domain", "general"),
|
||||
"author": meta.get("author", ""),
|
||||
"auto_match": meta.get("auto_match", {}),
|
||||
"schema_fields": list(schema.get("properties", {}).get("features", {}).get("items", {}).get("properties", {}).keys()),
|
||||
}
|
||||
|
||||
def list_skills(self) -> list[dict]:
|
||||
"""Return metadata for all discovered skills."""
|
||||
return [
|
||||
{
|
||||
"name": s["name"],
|
||||
"display_name": s["display_name"],
|
||||
"description": s["description"],
|
||||
"version": s["version"],
|
||||
"domain": s["domain"],
|
||||
}
|
||||
for s in self._cache.values()
|
||||
]
|
||||
|
||||
def get_skill(self, name: str) -> dict | None:
|
||||
"""Get a specific skill's full metadata."""
|
||||
return self._cache.get(name)
|
||||
|
||||
def reload(self):
|
||||
"""Re-scan the skills directory (for hot-reload)."""
|
||||
logger.info("[SKILL] 热重载 skills...")
|
||||
self._cache = {}
|
||||
self._scan()
|
||||
|
||||
|
||||
skill_manager = SkillManager()
|
||||
@@ -0,0 +1,66 @@
|
||||
# 用例导出服务:将用例集转换为 YAML / CSV 格式
|
||||
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
|
||||
import yaml
|
||||
|
||||
from server.services.testcase_engine.generator import tc_generator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TestCaseExporter:
|
||||
"""Export test case sets to YAML, CSV, or structured data."""
|
||||
|
||||
async def export(self, tc_set_id: str, fmt: str) -> tuple[bytes | str, str]:
|
||||
"""Export testcases in specified format. Returns (content, mime_type, filename)."""
|
||||
tc_set = await tc_generator.get_tc_set(tc_set_id)
|
||||
if not tc_set:
|
||||
raise ValueError(f"Test case set not found: {tc_set_id}")
|
||||
|
||||
cases = tc_set.get("cases", [])
|
||||
|
||||
if fmt == "yaml":
|
||||
return self._to_yaml(cases), "application/x-yaml", f"testcases_{tc_set_id}.yaml"
|
||||
elif fmt == "csv":
|
||||
return self._to_csv(cases), "text/csv", f"testcases_{tc_set_id}.csv"
|
||||
elif fmt == "json":
|
||||
import json
|
||||
return json.dumps(tc_set, ensure_ascii=False, indent=2), "application/json", f"testcases_{tc_set_id}.json"
|
||||
else:
|
||||
raise ValueError(f"Unsupported export format: {fmt}")
|
||||
|
||||
def _to_yaml(self, cases: list[dict]) -> str:
|
||||
"""Convert cases to YAML string."""
|
||||
output = {
|
||||
"testcases": cases,
|
||||
"metadata": {
|
||||
"total": len(cases),
|
||||
"exported_format": "yaml",
|
||||
},
|
||||
}
|
||||
return yaml.dump(output, allow_unicode=True, sort_keys=False, default_flow_style=False)
|
||||
|
||||
def _to_csv(self, cases: list[dict]) -> str:
|
||||
"""Convert cases to CSV string."""
|
||||
if not cases:
|
||||
return ""
|
||||
|
||||
# Flatten tags for CSV
|
||||
flattened = []
|
||||
for c in cases:
|
||||
row = {k: v for k, v in c.items() if k != "tags"}
|
||||
row["tags"] = ";".join(c.get("tags", []))
|
||||
flattened.append(row)
|
||||
|
||||
output = io.StringIO()
|
||||
fieldnames = ["id", "module", "feature", "case_title", "preconditions", "steps", "expected_result", "priority", "tags"]
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(flattened)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
tc_exporter = TestCaseExporter()
|
||||
@@ -0,0 +1,212 @@
|
||||
# 测试用例生成器:基于 IR 生成测试用例集
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from server.config import settings
|
||||
from server.core.llm_provider.router import router
|
||||
from server.services.ir_engine.generator import ir_generator
|
||||
|
||||
logger = logging.getLogger("testflow")
|
||||
|
||||
DEFAULT_GEN_PROMPT = """你是一位资深的软件测试工程师。基于以下中间表示(IR),请生成完整的测试用例集。
|
||||
|
||||
测试设计原则:
|
||||
1. 每个功能点至少覆盖一条正向用例和一条异常用例
|
||||
2. 对于 P0 功能点,额外覆盖边界值测试
|
||||
3. 用例步骤使用 Given-When-Then 结构
|
||||
4. 预期结果必须明确可验证
|
||||
|
||||
输出格式:纯 JSON 数组,每个测试用例包含以下字段:
|
||||
- id: 唯一标识符(字符串)
|
||||
- module: 模块名
|
||||
- feature: 功能点名称
|
||||
- case_title: 用例标题
|
||||
- preconditions: 前置条件
|
||||
- steps: 测试步骤(Given-When-Then 格式)
|
||||
- expected_result: 预期结果
|
||||
- priority: 用例优先级 (P0/P1/P2)
|
||||
- tags: 标签数组(如:["正向", "异常", "边界"])
|
||||
|
||||
## IR 内容:
|
||||
{{ ir_content }}
|
||||
|
||||
请只输出 JSON 数组,不要用 ```json 代码块包裹。"""
|
||||
|
||||
|
||||
class TestCaseGenerator:
|
||||
"""Generate test case sets from IR YAML using LLM."""
|
||||
|
||||
def __init__(self):
|
||||
self._tc_store: dict[str, dict] = {}
|
||||
|
||||
async def generate(self, ir_id: str, skill_name: str = "default") -> dict:
|
||||
"""Generate test cases from an IR version."""
|
||||
# Get IR content
|
||||
ir_version = await ir_generator.get_ir(ir_id)
|
||||
if not ir_version:
|
||||
return {"error": f"IR version not found: {ir_id}", "cases": []}
|
||||
|
||||
# Handle both old YAML and new pipeline JSON formats
|
||||
ir_content = ir_version.get("yaml_content", "")
|
||||
ir_json = ir_version.get("ir_json")
|
||||
|
||||
# If we have the new pipeline format (rules array), convert to testcases directly
|
||||
if ir_json and ir_json.get("rules"):
|
||||
logger.info("[TC] 从 %d 条 IR 规则转换测试用例...", len(ir_json["rules"]))
|
||||
cases = self._rules_to_cases(ir_json["rules"], ir_json.get("feature", ""))
|
||||
tc_set_id = str(uuid.uuid4())[:8]
|
||||
tc_set = self._store_tc(tc_set_id, ir_id, cases)
|
||||
# Log distribution
|
||||
p0 = sum(1 for c in cases if c.get("priority") == "P0")
|
||||
p1 = sum(1 for c in cases if c.get("priority") == "P1")
|
||||
logger.info("[TC] ✅ 生成 %d 条用例 (P0:%d P1:%d P2:%d)",
|
||||
len(cases), p0, p1, len(cases) - p0 - p1)
|
||||
return tc_set
|
||||
|
||||
# Fallback: use LLM with skill prompt
|
||||
if not ir_content:
|
||||
return {"error": "IR has no content", "cases": [], "tc_set_id": ""}
|
||||
|
||||
# Load skill prompt
|
||||
skill = ir_generator.load_skill(skill_name)
|
||||
prompt_template = skill.get("gen_cases_prompt", DEFAULT_GEN_PROMPT)
|
||||
prompt = prompt_template.replace("{{ ir_content }}", ir_content)
|
||||
|
||||
logger.info("[TC] 调用 LLM 生成测试用例 for IR %s with skill %s", ir_id, skill_name)
|
||||
|
||||
client = router.get_text_client()
|
||||
try:
|
||||
raw = client.chat(
|
||||
model=router.text_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.0,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Test case generation failed: %s", e)
|
||||
return {"error": str(e), "cases": [], "tc_set_id": ""}
|
||||
|
||||
# Parse JSON response
|
||||
cases = self._parse_json(raw)
|
||||
|
||||
# Create test case set
|
||||
tc_set_id = str(uuid.uuid4())[:8]
|
||||
tc_set = {
|
||||
"tc_set_id": tc_set_id,
|
||||
"ir_id": ir_id,
|
||||
"cases": cases,
|
||||
"case_count": len(cases),
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
self._tc_store[tc_set_id] = tc_set
|
||||
|
||||
# Save to output
|
||||
output_dir = settings.OUTPUT_DIR / ir_version.get("prd_id", "unknown") / "tc_sets"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
tc_path = output_dir / f"tc_{tc_set_id}.json"
|
||||
with open(tc_path, "w", encoding="utf-8") as f:
|
||||
json.dump(tc_set, f, ensure_ascii=False, indent=2)
|
||||
|
||||
usg = client.usage
|
||||
logger.info("Test cases generated: id=%s, count=%d, tokens: prompt=%d completion=%d total=%d",
|
||||
tc_set_id, len(cases), usg["prompt_tokens"], usg["completion_tokens"], usg["total_tokens"])
|
||||
|
||||
return tc_set
|
||||
|
||||
async def get_tc_set(self, tc_set_id: str) -> dict | None:
|
||||
"""Get a test case set by ID."""
|
||||
return self._tc_store.get(tc_set_id)
|
||||
|
||||
@staticmethod
|
||||
def _parse_json(raw: str) -> list[dict]:
|
||||
"""Parse JSON from LLM response, handling code fences."""
|
||||
text = raw.strip()
|
||||
if text.startswith("```json") or text.startswith("```"):
|
||||
first_nl = text.find("\n")
|
||||
if first_nl != -1:
|
||||
text = text[first_nl + 1:]
|
||||
if text.endswith("```"):
|
||||
text = text[:-3]
|
||||
text = text.strip()
|
||||
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
if isinstance(parsed, list):
|
||||
return parsed
|
||||
if isinstance(parsed, dict) and "cases" in parsed:
|
||||
return parsed["cases"]
|
||||
return [parsed] if isinstance(parsed, dict) else []
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Failed to parse test case JSON, returning empty")
|
||||
return []
|
||||
|
||||
def _rules_to_cases(self, rules: list[dict], feature: str) -> list[dict]:
|
||||
"""Convert pipeline IR rules to test case format. TC IDs: TC-{feature_id}-{seq}"""
|
||||
import re
|
||||
feature_id = re.sub(r'[^A-Z]', '', feature.upper())[:6] or "FEAT"
|
||||
if len(feature_id) < 2:
|
||||
feature_id = (feature_id + "FEAT")[:4]
|
||||
cases = []
|
||||
for i, rule in enumerate(rules):
|
||||
tc_id = f"TC-{feature_id}-{i+1:03d}"
|
||||
desc = rule.get("description", "")
|
||||
|
||||
# Build steps string from trigger + actions
|
||||
trigger = rule.get("trigger", {})
|
||||
conditions = trigger.get("conditions", [])
|
||||
cond_texts = []
|
||||
for c in conditions:
|
||||
unit = f" {c.get('unit', '')}" if c.get('unit') else ""
|
||||
cond_texts.append(f"{c.get('signal','?')} {c.get('operator','?')} {c.get('value','?')}{unit}")
|
||||
given = "Given " + (", ".join(cond_texts) if cond_texts else "触发条件满足")
|
||||
|
||||
actions = rule.get("actions", [])
|
||||
action_texts = []
|
||||
for a in actions:
|
||||
if a.get("type") == "user_interaction":
|
||||
action_texts.append(f"{a.get('description','')}('{a.get('content','')}')")
|
||||
else:
|
||||
action_texts.append(a.get("description", ""))
|
||||
then = "Then " + ("; ".join(action_texts) if action_texts else "执行动作")
|
||||
|
||||
steps = f"{given}\nWhen 条件触发\n{then}"
|
||||
|
||||
# Determine tags
|
||||
tags = ["正向"]
|
||||
priority = rule.get("priority", "P2")
|
||||
if priority == "P0":
|
||||
tags.append("冒烟")
|
||||
precond = rule.get("precondition", {})
|
||||
if precond:
|
||||
tags.append(f"{precond.get('app_type', '')} {precond.get('app_state', '')}".strip())
|
||||
|
||||
cases.append({
|
||||
"id": tc_id,
|
||||
"ir_rule_id": rule.get("rule_id", ""),
|
||||
"module": feature,
|
||||
"feature": rule.get("description", desc)[:40],
|
||||
"case_title": desc[:60],
|
||||
"preconditions": json.dumps(precond, ensure_ascii=False) if precond else "",
|
||||
"steps": steps,
|
||||
"expected_result": action_texts[-1] if action_texts else desc,
|
||||
"priority": priority,
|
||||
"tags": [t for t in tags if t],
|
||||
})
|
||||
return cases
|
||||
|
||||
def _store_tc(self, tc_set_id: str, ir_id: str, cases: list[dict]) -> dict:
|
||||
"""Store a test case set and return its metadata."""
|
||||
tc_set = {
|
||||
"tc_set_id": tc_set_id,
|
||||
"ir_id": ir_id,
|
||||
"cases": cases,
|
||||
"case_count": len(cases),
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
self._tc_store[tc_set_id] = tc_set
|
||||
return tc_set
|
||||
|
||||
|
||||
tc_generator = TestCaseGenerator()
|
||||
Reference in New Issue
Block a user