fix: harden backlog refine prompt scaffold and mixed-format parsing (#228)

djm81 · web-flow · commit 1870cfe62f46 · 2026-02-12T19:12:00.000+01:00
* fix: harden backlog refine prompt scaffold and parsing

* fix: normalize mixed notes parsing and boundary flushing

---------

Co-authored-by: Dominikus Nold &lt;djm81@users.noreply.github.com&gt;
diff --git a/openspec/CHANGE_ORDER.md b/openspec/CHANGE_ORDER.md
@@ -60,7 +60,7 @@ All entries in the table below are pending implementation.
 |--------|-------|---------------|----------|------------|
 | backlog-core | 01 | backlog-core-01-dependency-analysis-commands | [#116](https://github.com/nold-ai/specfact-cli/issues/116) | — |
 | backlog-core | 02 | backlog-core-02-interactive-issue-creation | [#173](https://github.com/nold-ai/specfact-cli/issues/173) | #116 (optional: #176, #177) |
-| backlog-core | 03 | backlog-core-03-refine-writeback-field-splitting | #225 | — |
+| backlog-core | 03 | backlog-core-03-refine-writeback-field-splitting | #225, #227 | — |
 
 ### backlog-scrum
 
diff --git a/openspec/changes/backlog-core-03-refine-writeback-field-splitting/CHANGE_VALIDATION.md b/openspec/changes/backlog-core-03-refine-writeback-field-splitting/CHANGE_VALIDATION.md
@@ -71,6 +71,19 @@ No public CLI signature changes or adapter method signature changes were introdu
 - `hatch test -- tests/unit/commands/test_backlog_commands.py -k TestParseRefinementOutputFields -v`: Pass (`5 passed`) after parser fix.
 - `hatch test -- tests/unit/adapters/test_ado_backlog_adapter.py tests/unit/adapters/test_github_backlog_adapter.py -v`: Pass (`35 passed`).
 
+## User Report Follow-up Validation (Prompt + Mixed Format Parsing, 2026-02-12)
+
+- `hatch test -- tests/unit/commands/test_backlog_commands.py -k mixed_heading_and_inline_notes -v`: Failing pre-implementation evidence captured.
+- `hatch test -- tests/unit/backlog/test_ai_refiner.py -k "expected_output_scaffold or omit_unknown_metadata_fields" -v`: Failing pre-implementation evidence captured.
+- `hatch test -- tests/unit/commands/test_backlog_commands.py -k TestParseRefinementOutputFields -v`: Pass (`6 passed`) after parser and prompt fixes.
+- `hatch test -- tests/unit/backlog/test_ai_refiner.py -k generate_refinement_prompt -v`: Pass (`5 passed`).
+- `hatch test -- tests/unit/adapters/test_ado_backlog_adapter.py tests/unit/adapters/test_github_backlog_adapter.py -v`: Pass (`35 passed`).
+
+## Review Follow-up Validation (Notes Duplication + Internal Heading Preservation, 2026-02-12)
+
+- `hatch test -- tests/unit/commands/test_backlog_commands.py -k "mixed_heading_and_inline_notes_preserves_description_before_notes or label_notes_with_internal_heading_keeps_heading_content" -v`: Failing pre-implementation evidence captured, then pass (`2 passed`) after parser fix.
+- `hatch test -- tests/unit/commands/test_backlog_commands.py -k TestParseRefinementOutputFields -v`: Pass (`7 passed`).
+
 ## Next Steps
 
 1. Complete implementation and tests per `tasks.md`.
diff --git a/openspec/changes/backlog-core-03-refine-writeback-field-splitting/TDD_EVIDENCE.md b/openspec/changes/backlog-core-03-refine-writeback-field-splitting/TDD_EVIDENCE.md
@@ -80,3 +80,64 @@
 - **Summary**:
   - `5 passed`
   - Includes regression coverage for label-only field blocks without `Description:`.
+
+## User Report Follow-up: Prompt Scaffold + Mixed Format Parsing
+
+### Pre-Implementation Failing Run
+
+- **Timestamp**: 2026-02-12T11:06:00+01:00
+- **Command**: `hatch test -- tests/unit/commands/test_backlog_commands.py -k mixed_heading_and_inline_notes -v`
+- **Result**: Failed
+- **Failure Summary**:
+  - `test_mixed_heading_and_inline_notes_preserves_description_before_notes` failed.
+  - Parser dropped pre-notes description narrative and kept only content starting at inline `**Notes**:`.
+
+- **Timestamp**: 2026-02-12T11:06:00+01:00
+- **Command**: `hatch test -- tests/unit/backlog/test_ai_refiner.py -k "expected_output_scaffold or omit_unknown_metadata_fields" -v`
+- **Result**: Failed
+- **Failure Summary**:
+  - Prompt did not include explicit output scaffold instructions.
+  - Prompt did not include instruction to omit unknown metadata fields/placeholders.
+
+### Post-Implementation Passing Run
+
+- **Timestamp**: 2026-02-12T11:08:00+01:00
+- **Command**: `hatch test -- tests/unit/commands/test_backlog_commands.py -k TestParseRefinementOutputFields -v`
+- **Result**: Passed
+- **Summary**:
+  - `6 passed`
+  - Includes mixed heading + inline notes regression.
+
+- **Timestamp**: 2026-02-12T11:08:00+01:00
+- **Command**: `hatch test -- tests/unit/backlog/test_ai_refiner.py -k generate_refinement_prompt -v`
+- **Result**: Passed
+- **Summary**:
+  - `5 passed`
+  - Includes prompt scaffold and metadata omission instruction coverage.
+
+## Review Follow-up: Notes Duplication + Internal Heading Truncation
+
+### Pre-Implementation Failing Run
+
+- **Timestamp**: 2026-02-12T17:47:00+01:00
+- **Command**: `hatch test -- tests/unit/commands/test_backlog_commands.py -k "mixed_heading_and_inline_notes_preserves_description_before_notes or label_notes_with_internal_heading_keeps_heading_content" -v`
+- **Result**: Failed
+- **Failure Summary**:
+  - `test_mixed_heading_and_inline_notes_preserves_description_before_notes` failed because raw `**Notes**:` markup and notes text were duplicated in `body_markdown`.
+  - `test_label_notes_with_internal_heading_keeps_heading_content` failed because notes content was truncated at internal `## Risks` heading.
+
+### Post-Implementation Passing Run
+
+- **Timestamp**: 2026-02-12T17:53:00+01:00
+- **Command**: `hatch test -- tests/unit/commands/test_backlog_commands.py -k "mixed_heading_and_inline_notes_preserves_description_before_notes or label_notes_with_internal_heading_keeps_heading_content" -v`
+- **Result**: Passed
+- **Summary**:
+  - `2 passed`
+  - Duplicate inline-notes markup removed from description output; internal notes headings preserved.
+
+- **Timestamp**: 2026-02-12T17:54:00+01:00
+- **Command**: `hatch test -- tests/unit/commands/test_backlog_commands.py -k TestParseRefinementOutputFields -v`
+- **Result**: Passed
+- **Summary**:
+  - `7 passed`
+  - Full parser regression suite passes including mixed-format and internal-heading cases.
diff --git a/openspec/changes/backlog-core-03-refine-writeback-field-splitting/proposal.md b/openspec/changes/backlog-core-03-refine-writeback-field-splitting/proposal.md
@@ -3,13 +3,15 @@
 ## Why
 
 
+
 `specfact backlog refine --write` currently applies the raw copilot response as `body_markdown` and does not parse structured refinement output back into canonical fields before adapter writeback. For Azure DevOps this causes `System.Description` to receive a verbatim payload containing labels like `Description`, `Acceptance Criteria`, `Story Points`, `Business Value`, `Priority`, `Area Path`, and provider markers instead of updating separate fields. GitHub can exhibit the same issue when copilot output uses label-style sections instead of markdown headings.
 
 This breaks the provider-aware contract implied by refinement prompts and produces low-quality remote item updates.
 
 ## What Changes
 
 
+
 - **MODIFY**: Backlog refine write path to parse structured refinement content into canonical fields (`description`, `acceptance_criteria`, `story_points`, `business_value`, `priority`, `work_item_type`) before writeback.
 - **MODIFY**: Normalize label-style refinement output to canonical markdown sections so both ADO and GitHub writeback paths behave deterministically.
 - **MODIFY**: ADO/GitHub writeback behavior to prefer parsed refined values over stale pre-refinement values when `--write` is used.
@@ -19,12 +21,23 @@ This breaks the provider-aware contract implied by refinement prompts and produc
 ## Capabilities
 - **backlog-refinement**: Provider-aware parsing and canonical field splitting for `specfact backlog refine --write`.
 
+
 ---
 
 ## Source Tracking
 
-<!-- source_repo: nold-ai/specfact-cli -->
+### Repository: nold-ai/specfact-cli
+
 - **GitHub Issue**: #225
 - **Issue URL**: <https://github.com/nold-ai/specfact-cli/issues/225>
 - **Last Synced Status**: proposed
 - **Sanitized**: false
+
+---
+
+### Repository: nold-ai/specfact-cli
+
+- **GitHub Issue**: #227
+- **Issue URL**: <https://github.com/nold-ai/specfact-cli/issues/227>
+- **Last Synced Status**: proposed
+- **Sanitized**: false
diff --git a/openspec/changes/backlog-core-03-refine-writeback-field-splitting/specs/backlog-refinement/spec.md b/openspec/changes/backlog-core-03-refine-writeback-field-splitting/specs/backlog-refinement/spec.md
@@ -48,6 +48,36 @@ The system SHALL parse structured refinement output into canonical fields before
 - **AND** parser fallback does not keep the entire raw labeled payload as `description`
 - **AND** `body_markdown` does not contain prompt labels verbatim.
 
+#### Scenario: Mixed heading and inline label formatting preserves description narrative
+
+- **GIVEN** a refined output that uses heading-style sections such as `## Description` and `## Acceptance Criteria`
+- **AND** the `## Description` section contains an inline label like `**Notes**:`
+- **WHEN** SpecFact parses the refinement output for writeback
+- **THEN** text before the inline label in `## Description` is preserved in `body_markdown`
+- **AND** label-capture does not swallow subsequent heading sections.
+
+#### Scenario: Prompt format contract includes canonical scaffold and metadata omission rule
+
+- **GIVEN** SpecFact generates a refinement prompt for IDE Copilot
+- **WHEN** prompt text is rendered for backlog refine
+- **THEN** it includes an explicit expected output scaffold (ordered canonical sections)
+- **AND** it instructs Copilot to omit unknown metadata fields (for example area/iteration path) instead of placeholders like "unspecified" or "provide ...".
+
+#### Scenario: Mixed heading description does not duplicate inline notes block
+
+- **GIVEN** a refined output with `## Description` narrative
+- **AND** an inline label block like `**Notes**:` appears inside that description section
+- **WHEN** SpecFact parses the refinement output for writeback
+- **THEN** description narrative is preserved without raw inline label markup
+- **AND** notes content appears only once in normalized `## Notes` output.
+
+#### Scenario: Label-style notes preserves internal non-boundary headings
+
+- **GIVEN** a label-style `Notes:` section includes internal headings such as `## Risks`
+- **WHEN** SpecFact parses notes/dependencies label blocks
+- **THEN** internal headings that are not canonical section boundaries are preserved as notes content
+- **AND** parser does not truncate notes at the internal heading line.
+
 #### Scenario: Refine command orchestration remains behaviorally consistent after decomposition
 
 - **GIVEN** `specfact backlog refine` supports initialization, filtering, export/import, interactive refinement, writeback, and summary flows
diff --git a/openspec/changes/backlog-core-03-refine-writeback-field-splitting/tasks.md b/openspec/changes/backlog-core-03-refine-writeback-field-splitting/tasks.md
@@ -27,6 +27,10 @@
 - [x] 4.7 Preserve heading-style `## Notes` / `## Dependencies` sections in parsed `body_markdown` for writeback
 - [x] 4.8 Match heading-style `## Notes` / `## Dependencies` sections case-insensitively during parser writeback normalization
 - [x] 4.9 Prevent label-only refinement output (without `Description:`) from leaking raw prompt labels into fallback description/body fields
+- [x] 4.10 Preserve mixed-format `## Description` narrative content when inline label-style markers (for example `**Notes**:`) appear before later heading sections
+- [x] 4.11 Strengthen Copilot prompt with explicit output scaffold and "omit unknown metadata fields" rule (no placeholder values)
+- [x] 4.12 Strip inline label markers from retained heading-style description content to avoid duplicated `Notes` blocks in normalized body output
+- [x] 4.13 Restrict label-block heading flush to canonical section boundaries so internal headings (for example `## Risks`) remain inside notes/dependencies content
 
 ## 5. Quality Gates
 
diff --git a/src/specfact_cli/backlog/ai_refiner.py b/src/specfact_cli/backlog/ai_refiner.py
@@ -172,6 +172,28 @@ def generate_refinement_prompt(
 9. Provider-aware formatting:
    - **GitHub**: Use markdown headings in body (## Section Name)
    - **ADO**: Use markdown headings in body (will be mapped to separate ADO fields during writeback)
+10. Omit unknown metadata fields instead of placeholders (do not emit values like "unspecified", "no info provided", or "provide area path")
+11. Keep `## Description` focused on narrative body content; do not place metadata labels in description text.
+
+Expected Output Scaffold (ordered):
+## Work Item Properties / Metadata
+- Story Points: <number, omit line if unknown>
+- Business Value: <number, omit line if unknown>
+- Priority: <number, omit line if unknown>
+- Work Item Type: <type, omit line if unknown>
+
+## Description
+<main story narrative/body only>
+
+## Acceptance Criteria
+- [ ] <criterion>
+
+## Notes
+<optional; include only for ambiguity/risk/dependency context>
+
+Metadata scaffold rules:
+- The `## Work Item Properties / Metadata` section is optional; include only when at least one metadata value is known.
+- Omit unknown metadata fields and do not emit placeholders.
 
 Return ONLY the refined backlog item body content in markdown format. Do not include any explanations or metadata."""
         return prompt.strip()
diff --git a/src/specfact_cli/modules/backlog/src/commands.py b/src/specfact_cli/modules/backlog/src/commands.py
@@ -1002,6 +1002,24 @@ def _build_refine_export_content(
     export_content += "7. Include story points, business value, priority, and work item type when available\n"
     export_content += "8. For high-complexity stories, suggest splitting when appropriate\n"
     export_content += "9. Follow provider-aware formatting guidance listed per item\n\n"
+    export_content += "**Expected Output Scaffold (ordered):**\n"
+    export_content += "```markdown\n"
+    export_content += "## Work Item Properties / Metadata\n"
+    export_content += "- Story Points: <number, omit line if unknown>\n"
+    export_content += "- Business Value: <number, omit line if unknown>\n"
+    export_content += "- Priority: <number, omit line if unknown>\n"
+    export_content += "- Work Item Type: <type, omit line if unknown>\n\n"
+    export_content += "## Description\n"
+    export_content += "<main story narrative/body only>\n\n"
+    export_content += "## Acceptance Criteria\n"
+    export_content += "- [ ] <criterion>\n\n"
+    export_content += "## Notes\n"
+    export_content += "<optional; include only for ambiguity/risk/dependency context>\n"
+    export_content += "```\n\n"
+    export_content += (
+        "Omit unknown metadata fields and never emit placeholders such as "
+        "`(unspecified)`, `no info provided`, or `provide area path`.\n\n"
+    )
     export_content += "---\n\n"
     comments_map = comments_by_item_id or {}
     template_map = template_guidance_by_item_id or {}
@@ -1526,13 +1544,26 @@ def _parse_refinement_output_fields(content: str) -> dict[str, Any]:
         if isinstance(value, int):
             parsed[key] = value
 
+    def _has_heading_section(section_name: str) -> bool:
+        return bool(
+            re.search(
+                rf"^##+\s+{re.escape(section_name)}\s*$",
+                normalized,
+                re.MULTILINE | re.IGNORECASE,
+            )
+        )
+
     def _extract_heading_section(section_name: str) -> str:
         pattern = rf"^##+\s+{re.escape(section_name)}\s*$\n(.*?)(?=^##|\Z)"
         match = re.search(pattern, normalized, re.MULTILINE | re.DOTALL | re.IGNORECASE)
         if not match:
             return ""
         return match.group(1).strip()
 
+    heading_description = _extract_heading_section("Description")
+    if heading_description and not (parsed.get("description") or "").strip():
+        parsed["description"] = heading_description
+
     # Then parse label-style blocks; explicit labels override heading heuristics.
     label_aliases = {
         "description": "description",
@@ -1547,11 +1578,24 @@ def _extract_heading_section(section_name: str) -> str:
         "iteration path": "iteration_path",
         "provider": "provider",
     }
+    canonical_heading_boundaries = {
+        *label_aliases.keys(),
+        "work item properties / metadata",
+        "work item properties",
+        "metadata",
+    }
     label_pattern = re.compile(r"^\s*(?:[-*]\s*)?(?:\*\*)?([A-Za-z][A-Za-z0-9 ()/_-]*?)(?:\*\*)?\s*:\s*(.*)\s*$")
     blocks: dict[str, str] = {}
     current_key: str | None = None
     current_lines: list[str] = []
 
+    def _is_canonical_heading_boundary(line: str) -> bool:
+        heading_match = re.match(r"^\s*##+\s+(.+?)\s*$", line)
+        if not heading_match:
+            return False
+        heading_name = re.sub(r"\s+", " ", heading_match.group(1).strip().strip("#")).lower()
+        return heading_name in canonical_heading_boundaries
+
     def _flush_current() -> None:
         nonlocal current_key, current_lines
         if current_key is None:
@@ -1562,6 +1606,10 @@ def _flush_current() -> None:
         current_lines = []
 
     for line in normalized.splitlines():
+        # Stop label-style block capture only at canonical section-heading boundaries.
+        if current_key is not None and _is_canonical_heading_boundary(line):
+            _flush_current()
+            continue
         match = label_pattern.match(line)
         if match:
             candidate = re.sub(r"\s+", " ", match.group(1).strip().lower())
@@ -1576,11 +1624,29 @@ def _flush_current() -> None:
             current_lines.append(line.rstrip())
     _flush_current()
 
-    if blocks and not blocks.get("description"):
+    if blocks and not blocks.get("description") and not _has_heading_section("Description"):
         # If label-style blocks are present but no explicit Description block exists,
         # do not keep the heading parser fallback description (it may contain raw labels).
         parsed.pop("description", None)
 
+    if _has_heading_section("Description") and not blocks.get("description") and parsed.get("description"):
+        # In mixed heading output, trim inline label-style suffix blocks from description
+        # to avoid duplicating notes/dependencies in normalized body output.
+        description_lines: list[str] = []
+        for line in str(parsed["description"]).splitlines():
+            inline_match = label_pattern.match(line)
+            if inline_match:
+                candidate = re.sub(r"\s+", " ", inline_match.group(1).strip().lower())
+                canonical = label_aliases.get(candidate)
+                if canonical and canonical != "description":
+                    break
+            description_lines.append(line.rstrip())
+        cleaned_heading_description = "\n".join(description_lines).strip()
+        if cleaned_heading_description:
+            parsed["description"] = cleaned_heading_description
+        else:
+            parsed.pop("description", None)
+
     if blocks.get("description"):
         parsed["description"] = blocks["description"]
     if blocks.get("acceptance_criteria"):
diff --git a/tests/unit/backlog/test_ai_refiner.py b/tests/unit/backlog/test_ai_refiner.py
@@ -90,6 +90,26 @@ def test_generate_refinement_prompt_mentions_no_comments_when_empty(
         prompt = refiner.generate_refinement_prompt(arbitrary_backlog_item, user_story_template, comments=[])
         assert "No comments found" in prompt
 
+    @beartype
+    def test_generate_refinement_prompt_includes_expected_output_scaffold(
+        self, refiner: BacklogAIRefiner, arbitrary_backlog_item: BacklogItem, user_story_template: BacklogTemplate
+    ) -> None:
+        """Prompt includes canonical output scaffold for Copilot consistency."""
+        prompt = refiner.generate_refinement_prompt(arbitrary_backlog_item, user_story_template)
+        assert "Expected Output Scaffold" in prompt
+        assert "## Work Item Properties / Metadata" in prompt
+        assert "## Description" in prompt
+        assert "## Acceptance Criteria" in prompt
+
+    @beartype
+    def test_generate_refinement_prompt_instructs_to_omit_unknown_metadata_fields(
+        self, refiner: BacklogAIRefiner, arbitrary_backlog_item: BacklogItem, user_story_template: BacklogTemplate
+    ) -> None:
+        """Prompt instructs omitting unknown metadata fields instead of placeholders."""
+        prompt = refiner.generate_refinement_prompt(arbitrary_backlog_item, user_story_template)
+        assert "omit unknown metadata fields" in prompt.lower()
+        assert "do not emit placeholders" in prompt.lower()
+
     @beartype
     def test_validate_and_score_complete_refinement(
         self, refiner: BacklogAIRefiner, arbitrary_backlog_item: BacklogItem, user_story_template: BacklogTemplate
diff --git a/tests/unit/commands/test_backlog_commands.py b/tests/unit/commands/test_backlog_commands.py
