Merge pull request 'fix: [bug] IR 覆盖率回归:Layer B 从 92.6% 降至 63% + Layer A 新 schema 错误 - 来自 #18 - Closes #57' (#59) from dev/issue-57-fix-coverage-regression into main
CI / test (push) Successful in 13s
CI / test (push) Successful in 13s
This commit was merged in pull request #59.
This commit is contained in:
@@ -880,7 +880,7 @@ def run_ensemble_semantic_index(doc: dict) -> dict:
|
|||||||
if v:
|
if v:
|
||||||
print(f" {k}: {len(v)} 个问题")
|
print(f" {k}: {len(v)} 个问题")
|
||||||
|
|
||||||
# Feedback retry: re-run with coverage feedback (up to 2 retries)
|
# Feedback retry: re-run with coverage feedback (up to 2 retries, quality-gated)
|
||||||
retry_count = 0
|
retry_count = 0
|
||||||
while retry_count < 2:
|
while retry_count < 2:
|
||||||
feedback = _build_coverage_feedback(gaps)
|
feedback = _build_coverage_feedback(gaps)
|
||||||
@@ -889,6 +889,10 @@ def run_ensemble_semantic_index(doc: dict) -> dict:
|
|||||||
retry_count += 1
|
retry_count += 1
|
||||||
print(f"\n 覆盖反馈重试 #{retry_count} (feedback长度={len(feedback)}字符)...", flush=True)
|
print(f"\n 覆盖反馈重试 #{retry_count} (feedback长度={len(feedback)}字符)...", flush=True)
|
||||||
try:
|
try:
|
||||||
|
# record pre-retry coverage to gate quality
|
||||||
|
pre_warnings = len(gaps.get("coverage_warnings", []))
|
||||||
|
pre_missing_rows = len(gaps.get("missing_table_rows", []))
|
||||||
|
|
||||||
retry_prompt = build_prompt(doc, feedback, all_paths)
|
retry_prompt = build_prompt(doc, feedback, all_paths)
|
||||||
print(f" 重试 prompt 长度: {len(retry_prompt)} 字符", flush=True)
|
print(f" 重试 prompt 长度: {len(retry_prompt)} 字符", flush=True)
|
||||||
retry_result = call_llm(retry_prompt, max_retries=1, temperature=0.3)
|
retry_result = call_llm(retry_prompt, max_retries=1, temperature=0.3)
|
||||||
@@ -902,15 +906,28 @@ def run_ensemble_semantic_index(doc: dict) -> dict:
|
|||||||
if src.get("section"):
|
if src.get("section"):
|
||||||
retry_sections.add(src["section"])
|
retry_sections.add(src["section"])
|
||||||
print(f" 重试新增 sections: {sorted(retry_sections)}", flush=True)
|
print(f" 重试新增 sections: {sorted(retry_sections)}", flush=True)
|
||||||
semantic_indices.append(retry_result)
|
# Quality gate: only include retry if it improves coverage
|
||||||
merged = ensemble_merge(semantic_indices)
|
trial_indices = semantic_indices + [retry_result]
|
||||||
merged["ensemble_temperatures"] = list(temperatures) + [f"feedback_retry_{retry_count}"]
|
trial_merged = ensemble_merge(trial_indices)
|
||||||
passed, gaps = _quick_validate(merged, doc, all_paths)
|
trial_passed, trial_gaps = _quick_validate(trial_merged, doc, all_paths)
|
||||||
merged["validation_passed"] = passed
|
trial_warnings = len(trial_gaps.get("coverage_warnings", []))
|
||||||
merged["validation_gaps"] = {
|
trial_missing = len(trial_gaps.get("missing_table_rows", []))
|
||||||
k: v for k, v in gaps.items() if v
|
if trial_warnings < pre_warnings or trial_missing < pre_missing_rows:
|
||||||
}
|
semantic_indices.append(retry_result)
|
||||||
print(f" 重试后验证: {'PASS' if passed else 'GAPS FOUND'}", flush=True)
|
merged = trial_merged
|
||||||
|
passed, gaps = trial_passed, trial_gaps
|
||||||
|
merged["ensemble_temperatures"] = list(temperatures) + [f"feedback_retry_{retry_count}"]
|
||||||
|
merged["validation_passed"] = passed
|
||||||
|
merged["validation_gaps"] = {
|
||||||
|
k: v for k, v in gaps.items() if v
|
||||||
|
}
|
||||||
|
print(f" 重试后验证 (已采纳): {'PASS' if passed else 'GAPS FOUND'} "
|
||||||
|
f"(warnings {pre_warnings}→{trial_warnings}, "
|
||||||
|
f"missing_rows {pre_missing_rows}→{trial_missing})", flush=True)
|
||||||
|
else:
|
||||||
|
print(f" 重试结果未提升覆盖率,丢弃 "
|
||||||
|
f"(warnings {pre_warnings}→{trial_warnings}, "
|
||||||
|
f"missing_rows {pre_missing_rows}→{trial_missing})", flush=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f" 覆盖反馈重试失败: {e}", flush=True)
|
print(f" 覆盖反馈重试失败: {e}", flush=True)
|
||||||
import traceback
|
import traceback
|
||||||
|
|||||||
@@ -170,8 +170,11 @@ def _normalize_rule(rule: dict) -> dict:
|
|||||||
}]
|
}]
|
||||||
|
|
||||||
# Ensure table/text sources have a section field (defensive against LLM omission)
|
# Ensure table/text sources have a section field (defensive against LLM omission)
|
||||||
|
# Also normalize invalid source types (LLM hallucinations like function_unit_description)
|
||||||
sources = rule.get("sources", [])
|
sources = rule.get("sources", [])
|
||||||
if sources:
|
if sources:
|
||||||
|
valid_types = {"table", "text", "logic_tree"}
|
||||||
|
|
||||||
# try to infer a default section from sibling sources or the rule path
|
# try to infer a default section from sibling sources or the rule path
|
||||||
default_section = ""
|
default_section = ""
|
||||||
for s in sources:
|
for s in sources:
|
||||||
@@ -186,6 +189,10 @@ def _normalize_rule(rule: dict) -> dict:
|
|||||||
|
|
||||||
for src in sources:
|
for src in sources:
|
||||||
stype = src.get("type", "")
|
stype = src.get("type", "")
|
||||||
|
# Normalize invalid source types to "text"
|
||||||
|
if stype and stype not in valid_types:
|
||||||
|
src["type"] = "text"
|
||||||
|
stype = "text"
|
||||||
if stype in ("table", "text"):
|
if stype in ("table", "text"):
|
||||||
if not src.get("section"):
|
if not src.get("section"):
|
||||||
src["section"] = default_section
|
src["section"] = default_section
|
||||||
|
|||||||
@@ -511,3 +511,18 @@ class TestNormalizeRule:
|
|||||||
}
|
}
|
||||||
normalized = _normalize_rule(rule)
|
normalized = _normalize_rule(rule)
|
||||||
assert "section" not in normalized["sources"][0]
|
assert "section" not in normalized["sources"][0]
|
||||||
|
|
||||||
|
def test_normalize_source_invalid_type(self):
|
||||||
|
"""Invalid source types (LLM hallucinations) are normalized to text."""
|
||||||
|
rule = {
|
||||||
|
"trigger": {"conditions": [{"signal": "x", "operator": "==", "value": "1"}]},
|
||||||
|
"sources": [
|
||||||
|
{"type": "function_unit_description", "text_snippet": "desc",
|
||||||
|
"section": "3.1 功能"},
|
||||||
|
{"type": "unknown_type", "text_snippet": "also invalid"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
normalized = _normalize_rule(rule)
|
||||||
|
assert normalized["sources"][0]["type"] == "text"
|
||||||
|
assert normalized["sources"][1]["type"] == "text"
|
||||||
|
assert normalized["sources"][0]["section"] == "3.1 功能"
|
||||||
|
|||||||
Reference in New Issue
Block a user