init the project

server/services/prd_manager/service.py

@@ -0,0 +1,260 @@
+# PRD 管理服务：上传、解析、版本快照
+
+import json
+import logging
+import os
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from server.config import settings
+from server.services.prd_manager.word_parser import WordParser
+from server.services.prd_manager.image_parser import ImageParser
+
+logger = logging.getLogger("testflow")
+RATE_LIMIT_DELAY = 0.5
+
+
+class PRDService:
+    """PRD management: handle file upload, text extraction, and version snapshots."""
+
+    def __init__(self):
+        self.output_dir = settings.OUTPUT_DIR
+        os.makedirs(self.output_dir, exist_ok=True)
+        self._prds: dict[str, dict] = {}
+
+    async def upload_and_parse(self, filename: str, content: bytes) -> dict:
+        prd_id = str(uuid.uuid4())[:8]
+        basename = Path(filename).stem
+        ext = Path(filename).suffix.lower()
+
+        upload_dir = self.output_dir / prd_id
+        os.makedirs(upload_dir, exist_ok=True)
+        filepath = upload_dir / filename
+
+        t0 = time.time()
+        with open(filepath, "wb") as f:
+            f.write(content)
+
+        logger.info("[UPLOAD] 收到文件: %s (%.1f KB, %s)", filename, len(content) / 1024, ext)
+
+        if ext in (".txt", ".md"):
+            text = content.decode("utf-8", errors="replace")
+            sections = self._parse_markdown_sections(text)
+            parsed = {
+                "source": str(filepath), "sections": sections,
+                "image_sources": {}, "image_analysis": [], "full_text": text,
+            }
+            logger.info("[PARSE] %.4s %s → %d 章节, %d 字符, %.1fs",
+                         prd_id, ext, len(sections), len(text), time.time() - t0)
+
+        elif ext == ".docx":
+            logger.info("[PARSE] %.4s → 开始解析 .docx...", prd_id)
+            parsed = await self._parse_docx(str(filepath), upload_dir)
+            logger.info("[PARSE] %.4s .docx 完成: %d 章节, %.1fs",
+                         prd_id, len(parsed.get("sections", [])), time.time() - t0)
+
+        elif ext == ".pdf":
+            logger.info("[PARSE] %.4s → 开始解析 .pdf...", prd_id)
+            parsed = await self._parse_pdf(str(filepath), upload_dir)
+            logger.info("[PARSE] %.4s .pdf 完成: %d 页, %.1fs",
+                         prd_id, len(parsed.get("sections", [])), time.time() - t0)
+        else:
+            raise ValueError(f"Unsupported file format: {ext}")
+
+        parsed_path = upload_dir / f"{basename}_parsed.json"
+        with open(parsed_path, "w", encoding="utf-8") as f:
+            json.dump(parsed, f, ensure_ascii=False, indent=2)
+
+        prd_version = {
+            "prd_id": prd_id, "filename": filename,
+            "uploaded_at": datetime.now(timezone.utc).isoformat(), "status": "ready",
+            "parsed_path": str(parsed_path), "full_text": parsed.get("full_text", ""),
+            "sections": parsed.get("sections", []), "images": parsed.get("image_analysis", []),
+        }
+        self._prds[prd_id] = prd_version
+
+        n_paras = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "para")
+        n_tables = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "table")
+        logger.info("[UPLOAD] %.4s 解析完成: %d章节 %d段落 %d表格 %d图片 → %s",
+                     prd_id, len(parsed.get("sections", [])), n_paras, n_tables,
+                     len(parsed.get("image_analysis", [])), parsed_path)
+        return prd_version
+
+    async def upload_quick(self, filename: str, content: bytes) -> dict:
+        """Quick save — returns immediately, parsing happens in background."""
+        prd_id = str(uuid.uuid4())[:8]
+        upload_dir = self.output_dir / prd_id
+        os.makedirs(upload_dir, exist_ok=True)
+        filepath = upload_dir / filename
+        with open(filepath, "wb") as f:
+            f.write(content)
+
+        logger.info("[UPLOAD] %.4s 快速保存: %s (%.1f KB)", prd_id, filename, len(content) / 1024)
+
+        prd_version = {
+            "prd_id": prd_id, "filename": filename,
+            "uploaded_at": datetime.now(timezone.utc).isoformat(),
+            "status": "parsing", "full_text": "", "sections": [], "images": [],
+            "parsed_path": "", "filepath": str(filepath),
+        }
+        self._prds[prd_id] = prd_version
+        return prd_version
+
+    async def parse_async(self, prd_id: str, filename: str, content: bytes):
+        """Full parse (text + images + vision) — called in background thread."""
+        basename = Path(filename).stem
+        ext = Path(filename).suffix.lower()
+        upload_dir = self.output_dir / prd_id
+        filepath = upload_dir / filename
+        t0 = time.time()
+
+        # Parse
+        if ext in (".txt", ".md"):
+            text = content.decode("utf-8", errors="replace")
+            sections = self._parse_markdown_sections(text)
+            parsed = {"source": str(filepath), "sections": sections,
+                      "image_sources": {}, "image_analysis": [], "full_text": text}
+        elif ext == ".docx":
+            parsed = await self._parse_docx(str(filepath), upload_dir)
+        elif ext == ".pdf":
+            parsed = await self._parse_pdf(str(filepath), upload_dir)
+        else:
+            self.set_status(prd_id, "error", f"Unsupported format: {ext}")
+            return
+
+        # Save parsed
+        parsed_path = upload_dir / f"{basename}_parsed.json"
+        with open(parsed_path, "w", encoding="utf-8") as f:
+            json.dump(parsed, f, ensure_ascii=False, indent=2)
+
+        # Update in-memory
+        prd = self._prds.get(prd_id, {})
+        prd.update({
+            "status": "ready", "parsed_path": str(parsed_path),
+            "full_text": parsed.get("full_text", ""),
+            "sections": parsed.get("sections", []),
+            "images": parsed.get("image_analysis", []),
+        })
+
+        n_paras = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "para")
+        n_tables = sum(1 for s in parsed.get("sections", []) for b in s.get("blocks", []) if b["type"] == "table")
+        logger.info("[UPLOAD] %.4s 后台解析完成: %d章节 %d段落 %d表格 %d图片, %.1fs",
+                     prd_id, len(parsed.get("sections", [])), n_paras, n_tables,
+                     len(parsed.get("image_analysis", [])), time.time() - t0)
+
+    def set_status(self, prd_id: str, status: str, error: str = ""):
+        if prd_id in self._prds:
+            self._prds[prd_id]["status"] = status
+            if error:
+                self._prds[prd_id]["error"] = error
+
+    async def get_status(self, prd_id: str) -> dict | None:
+        prd = self._prds.get(prd_id)
+        if not prd:
+            return None
+        return {"prd_id": prd_id, "status": prd.get("status", "ready"),
+                "error": prd.get("error", ""),
+                "sections_count": len(prd.get("sections", [])),
+                "images_count": len(prd.get("images", []))}
+
+    async def get_prd(self, prd_id: str) -> dict | None:
+        return self._prds.get(prd_id)
+
+    # ---- internal parsers ----
+
+    async def _parse_docx(self, filepath: str, output_dir: Path) -> dict:
+        word = WordParser(filepath)
+        sections, image_sources = word.extract_sections()
+        full_text = word.extract_full_text()
+
+        images_dir = output_dir / "images"
+        images = word.extract_images(str(images_dir))
+        image_analysis = []
+        n_chart_images = 0
+
+        if images:
+            parser = ImageParser()
+            logger.info("[IMAGE] %.4s 发现 %d 张图片，开始视觉分析...", output_dir.parent.name, len(images))
+            for i, img in enumerate(images):
+                t_img = time.time()
+                logger.info("[IMAGE] [%d/%d] rid=%s, 调用 Qwen VL...", i + 1, len(images), img["rid"])
+                result = parser.parse_image(img["path"])
+                if result is None:
+                    result = {"type": "other", "description": ""}
+                result["rid"] = img["rid"]
+                result["path"] = img["path"]
+                if img["rid"] in image_sources:
+                    result["context"] = image_sources[img["rid"]]
+                image_analysis.append(result)
+                if result.get("type") in ("flowchart", "architecture", "state", "sequence", "activity"):
+                    n_chart_images += 1
+                logger.info("[IMAGE] [%d/%d] rid=%s 完成，类型=%s (%.1fs)",
+                             i + 1, len(images), img["rid"], result.get("type", "?"), time.time() - t_img)
+                if i < len(images) - 1:
+                    time.sleep(RATE_LIMIT_DELAY)
+
+            usg = parser.usage
+            logger.info("[IMAGE] %.4s 图片分析完成: %d张, 图表类%d张, tokens(p=%d c=%d t=%d)",
+                         output_dir.parent.name, len(images), n_chart_images,
+                         usg["prompt_tokens"], usg["completion_tokens"], usg["total_tokens"])
+        else:
+            logger.info("[IMAGE] %.4s 文档无图片", output_dir.parent.name)
+
+        return {
+            "source": filepath, "sections": sections,
+            "image_sources": image_sources, "image_analysis": image_analysis,
+            "full_text": full_text,
+        }
+
+    def _parse_markdown_sections(self, text: str) -> list[dict]:
+        import re
+        sections: list[dict] = []
+        current_source = "正文"
+        current_blocks: list[dict] = []
+        para_idx = 0
+
+        for line in text.split("\n"):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            heading_match = re.match(r"^(#{1,6})\s+(.+)", stripped)
+            if heading_match:
+                if current_blocks:
+                    sections.append({"source": current_source, "blocks": current_blocks, "images": []})
+                    current_blocks = []
+                level = len(heading_match.group(1))
+                current_source = f"{'#' * level} {heading_match.group(2)}"
+                continue
+            if "|" in stripped and stripped.count("|") >= 2:
+                cells = [c.strip() for c in stripped.split("|") if c.strip()]
+                if cells:
+                    current_blocks.append({
+                        "type": "table",
+                        "table": len([b for b in current_blocks if b["type"] == "table"]) + 1,
+                        "headers": cells,
+                        "rows": [{"columns": [{"name": c, "row": 1, "col": i + 1, "text": c} for i, c in enumerate(cells)]}],
+                    })
+                continue
+            para_idx += 1
+            current_blocks.append({"type": "para", "index": para_idx, "text": stripped})
+
+        if current_blocks:
+            sections.append({"source": current_source, "blocks": current_blocks, "images": []})
+        return sections
+
+    async def _parse_pdf(self, filepath: str, output_dir: Path) -> dict:
+        from PyPDF2 import PdfReader
+        reader = PdfReader(filepath)
+        full_text_parts = []
+        for i, page in enumerate(reader.pages):
+            text = page.extract_text()
+            if text:
+                full_text_parts.append(text)
+        full_text = "\n".join(full_text_parts)
+        sections = [{"source": "PDF 内容", "blocks": [{"type": "para", "index": i + 1, "text": t} for i, t in enumerate(full_text_parts)], "images": []}]
+        return {"source": filepath, "sections": sections, "image_sources": {}, "image_analysis": [], "full_text": full_text}
+
+
+prd_service = PRDService()