feat(runtime): implement single-article consolidation runtime and modularize codebase

src/runtime/__init__.py

@@ -0,0 +1 @@
+"""Article Consolidation Runtime namespace."""

src/runtime/candidate/equivalence.py

@@ -0,0 +1,46 @@
+"""Non-destructive candidate equivalence mapping using difflib.SequenceMatcher without regex."""
+
+from __future__ import annotations
+
+import difflib
+import unicodedata
+from typing import List
+
+from src.runtime.candidate.models import CandidateObject
+
+
+def normalize_text_for_comparison(text: str) -> str:
+    """Normalizes Unicode text and collapses whitespace without regular expressions."""
+    if not text:
+        return ""
+    norm = unicodedata.normalize("NFKC", text)
+    # Split on standard whitespace and rejoin with single space
+    return " ".join(norm.split()).lower()
+
+
+def compute_sequence_similarity(a: str, b: str) -> float:
+    """Computes character/token sequence ratio using standard library difflib."""
+    norm_a = normalize_text_for_comparison(a)
+    norm_b = normalize_text_for_comparison(b)
+    if not norm_a or not norm_b:
+        return 0.0
+    if norm_a == norm_b:
+        return 1.0
+    return difflib.SequenceMatcher(None, norm_a, norm_b).ratio()
+
+
+def map_candidate_equivalences(
+    backbone_candidates: List[CandidateObject],
+    other_candidates: List[CandidateObject],
+    similarity_threshold: float = 0.85,
+) -> None:
+    """Non-destructively annotates candidate objects with equivalent candidate IDs from other extractors."""
+    for b_cand in backbone_candidates:
+        for o_cand in other_candidates:
+            if b_cand.type == o_cand.type:
+                sim = compute_sequence_similarity(b_cand.text, o_cand.text)
+                if sim >= similarity_threshold:
+                    if o_cand.id not in b_cand.equivalent_ids:
+                        b_cand.equivalent_ids.append(o_cand.id)
+                    if b_cand.id not in o_cand.equivalent_ids:
+                        o_cand.equivalent_ids.append(b_cand.id)

src/runtime/candidate/models.py

@@ -0,0 +1,57 @@
+"""Candidate element data models and repair structures."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class CandidateType(str, Enum):
+    TITLE = "title"
+    SUBTITLE = "subtitle"
+    AUTHOR = "author"
+    DATE = "date"
+    BLOCK = "paragraph"
+    HEADING = "heading"
+    LIST_ITEM = "list_item"
+    QUOTE = "quote"
+    LINK = "link"
+    IMAGE = "image"
+
+
+class RepairCategory(str, Enum):
+    ENCODING = "encoding"
+    UNICODE = "unicode"
+    SPACING = "spacing"
+    PUNCTUATION_CORRUPTION = "punctuation_corruption"
+    OBVIOUS_TYPO = "obvious_typo"
+
+
+@dataclass
+class CandidateObject:
+    id: str  # opaque candidate ID (e.g. "blk_001", "img_001")
+    type: str  # paragraph, heading, list_item, quote, title, subtitle, author, link, image
+    text: str  # normalized text content
+    extractor: str  # trafilatura, newspaper4k, readability
+    position: int  # 0-indexed position within extractor stream
+    level: Optional[int] = None  # for headings (1..6)
+    href: Optional[str] = None  # for links
+    src: Optional[str] = None  # for images
+    alt: Optional[str] = None  # for images
+    caption: Optional[str] = None  # for images
+    equivalent_ids: List[str] = field(
+        default_factory=list
+    )  # equivalent candidates across extractors
+    extra_metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class TextRepair:
+    target_id: str
+    original: str
+    replacement: str
+    category: str
+    rationale: str
+    applied: bool = False
+    decision: str = "pending"  # approved, rejected

src/runtime/candidate/parser.py

@@ -0,0 +1,306 @@
+"""Structural candidate parser using DOM, CommonMark AST, and JSON-LD without regex."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+import marko
+from marko.block import Heading, ListItem, Paragraph, Quote
+from marko.block import List as MarkoList
+
+from src.runtime.candidate.equivalence import map_candidate_equivalences
+from src.runtime.candidate.models import CandidateObject
+from src.tools.language import detect_language
+
+
+def resolve_canonical_source_url(article_dict: Dict[str, Any]) -> str:
+    """Normative priorities: trafilatura.canonical_url -> input_meta.url -> crawled_url."""
+    traf = article_dict.get("trafilatura")
+    if isinstance(traf, dict) and traf.get("canonical_url"):
+        url = traf["canonical_url"].strip()
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+
+    input_meta = article_dict.get("input_meta")
+    if isinstance(input_meta, dict) and input_meta.get("url"):
+        url = input_meta["url"].strip()
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+
+    crawled = article_dict.get("crawled_url")
+    if (
+        crawled
+        and isinstance(crawled, str)
+        and (crawled.startswith("http://") or crawled.startswith("https://"))
+    ):
+        return crawled.strip()
+
+    raise ValueError("MISSING_SOURCE_URL: No valid HTTP/HTTPS source URL found in article input.")
+
+
+def parse_metadata_candidates(article_dict: Dict[str, Any]) -> Dict[str, List[Dict[str, str]]]:
+    """Extracts title, subtitle, and author candidates without delimiter-based author splitting."""
+    title_candidates: List[Dict[str, str]] = []
+    subtitle_candidates: List[Dict[str, str]] = []
+    author_candidates: List[Dict[str, str]] = []
+
+    # 1. Titles
+    input_meta = article_dict.get("input_meta", {})
+    if isinstance(input_meta, dict) and input_meta.get("titulo"):
+        title_candidates.append(
+            {
+                "candidate_id": "title_meta",
+                "source": "input_meta.titulo",
+                "text": input_meta["titulo"].strip(),
+            }
+        )
+
+    for ext in ["trafilatura", "newspaper4k", "readability"]:
+        data = article_dict.get(ext)
+        if isinstance(data, dict) and data.get("title"):
+            title_candidates.append(
+                {
+                    "candidate_id": f"title_{ext}",
+                    "source": f"{ext}.title",
+                    "text": data["title"].strip(),
+                }
+            )
+
+    # 2. Subtitles
+    if isinstance(input_meta, dict) and input_meta.get("subtitulo"):
+        subtitle_candidates.append(
+            {
+                "candidate_id": "subtitle_meta",
+                "source": "input_meta.subtitulo",
+                "text": input_meta["subtitulo"].strip(),
+            }
+        )
+
+    for ext in ["trafilatura", "newspaper4k", "readability"]:
+        data = article_dict.get(ext)
+        if isinstance(data, dict) and data.get("description"):
+            subtitle_candidates.append(
+                {
+                    "candidate_id": f"subtitle_{ext}",
+                    "source": f"{ext}.description",
+                    "text": data["description"].strip(),
+                }
+            )
+
+    # 3. Authors (strictly forbidding delimiter splitting by comma, slash, etc.)
+    for ext in ["trafilatura", "newspaper4k", "readability"]:
+        data = article_dict.get(ext)
+        if isinstance(data, dict):
+            author_val = data.get("author") or data.get("authors")
+            if author_val:
+                if isinstance(author_val, list):
+                    for a_idx, a_name in enumerate(author_val):
+                        if a_name and str(a_name).strip():
+                            author_candidates.append(
+                                {
+                                    "candidate_id": f"author_{ext}_{a_idx}",
+                                    "source": f"{ext}.authors[{a_idx}]",
+                                    "text": str(a_name).strip(),
+                                }
+                            )
+                elif isinstance(author_val, str) and author_val.strip():
+                    author_candidates.append(
+                        {
+                            "candidate_id": f"author_{ext}",
+                            "source": f"{ext}.author",
+                            "text": author_val.strip(),
+                        }
+                    )
+
+    return {
+        "title_candidates": title_candidates,
+        "subtitle_candidates": subtitle_candidates,
+        "author_candidates": author_candidates,
+    }
+
+
+def parse_raw_text_into_candidates(text: str, extractor: str) -> List[CandidateObject]:
+    """Parses raw extractor body text into candidate blocks using CommonMark AST."""
+    if not text or not text.strip():
+        return []
+
+    parsed_doc = marko.parse(text)
+    candidates: List[CandidateObject] = []
+    order_idx = 0
+
+    for child in parsed_doc.children:
+        if isinstance(child, Heading):
+            heading_text = _extract_plain_text(child).strip()
+            if heading_text:
+                order_idx += 1
+                cand = CandidateObject(
+                    id=f"{extractor}_blk_{order_idx:03d}",
+                    type="heading",
+                    text=heading_text,
+                    extractor=extractor,
+                    position=order_idx,
+                    level=child.level,
+                )
+                candidates.append(cand)
+        elif isinstance(child, Paragraph):
+            p_text = _extract_plain_text(child).strip()
+            if p_text:
+                order_idx += 1
+                cand = CandidateObject(
+                    id=f"{extractor}_blk_{order_idx:03d}",
+                    type="paragraph",
+                    text=p_text,
+                    extractor=extractor,
+                    position=order_idx,
+                )
+                candidates.append(cand)
+        elif isinstance(child, MarkoList):
+            for item in child.children:
+                if isinstance(item, ListItem):
+                    item_text = _extract_plain_text(item).strip()
+                    if item_text:
+                        order_idx += 1
+                        cand = CandidateObject(
+                            id=f"{extractor}_blk_{order_idx:03d}",
+                            type="list_item",
+                            text=item_text,
+                            extractor=extractor,
+                            position=order_idx,
+                        )
+                        candidates.append(cand)
+        elif isinstance(child, Quote):
+            quote_text = _extract_plain_text(child).strip()
+            if quote_text:
+                order_idx += 1
+                cand = CandidateObject(
+                    id=f"{extractor}_blk_{order_idx:03d}",
+                    type="quote",
+                    text=quote_text,
+                    extractor=extractor,
+                    position=order_idx,
+                )
+                candidates.append(cand)
+
+    # Fallback to simple paragraph split if AST produced zero children
+    if not candidates:
+        paragraphs = text.split("\n\n")
+        for p in paragraphs:
+            p_clean = " ".join(p.split()).strip()
+            if p_clean:
+                order_idx += 1
+                candidates.append(
+                    CandidateObject(
+                        id=f"{extractor}_blk_{order_idx:03d}",
+                        type="paragraph",
+                        text=p_clean,
+                        extractor=extractor,
+                        position=order_idx,
+                    )
+                )
+
+    return candidates
+
+
+def _extract_plain_text(element: Any) -> str:
+    if hasattr(element, "children"):
+        if isinstance(element.children, str):
+            return element.children
+        if isinstance(element.children, list):
+            return "".join(_extract_plain_text(c) for c in element.children)
+    return str(getattr(element, "text", ""))
+
+
+def build_candidates_payload(article_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """Builds the complete candidates payload conforming to candidates-payload.schema.json."""
+    selected_ext = article_dict.get("selected_extractor")
+    if not selected_ext:
+        raise ValueError("MISSING_SELECTED_EXTRACTOR: 'selected_extractor' is required.")
+
+    if selected_ext not in {"trafilatura", "newspaper4k", "readability"}:
+        raise ValueError(f"INVALID_SELECTED_EXTRACTOR: '{selected_ext}' is not a valid extractor.")
+
+    selected_data = article_dict.get(selected_ext)
+    if not isinstance(selected_data, dict):
+        raise ValueError(f"SELECTED_EXTRACTOR_UNAVAILABLE: '{selected_ext}' payload is missing.")
+
+    body_text = (
+        selected_data.get("body_text")
+        or selected_data.get("text")
+        or selected_data.get("cleaned_text")
+    )
+    if not body_text or not body_text.strip():
+        raise ValueError("MISSING_CONTENT: Selected extractor has empty body content.")
+
+    # Parse backbone candidates
+    backbone_candidates = parse_raw_text_into_candidates(body_text, selected_ext)
+    if not backbone_candidates:
+        raise ValueError("MISSING_CONTENT: Zero candidates parsed from body text.")
+
+    # Parse alternative extractors for equivalence mapping
+    for other_ext in ["trafilatura", "newspaper4k", "readability"]:
+        if other_ext != selected_ext:
+            other_data = article_dict.get(other_ext)
+            if isinstance(other_data, dict):
+                o_text = (
+                    other_data.get("body_text")
+                    or other_data.get("text")
+                    or other_data.get("cleaned_text")
+                )
+                if o_text:
+                    other_candidates = parse_raw_text_into_candidates(o_text, other_ext)
+                    map_candidate_equivalences(backbone_candidates, other_candidates)
+
+    # Detect language
+    lang_res = detect_language(body_text)
+    lang = lang_res[0] if isinstance(lang_res, tuple) else str(lang_res)
+
+    # Metadata candidates
+    metadata_cand = parse_metadata_candidates(article_dict)
+    if not metadata_cand["title_candidates"]:
+        raise ValueError("MISSING_TITLE_CANDIDATE: No title candidate found across input sources.")
+
+    # Build blocks
+    block_candidates = []
+    for idx, c in enumerate(backbone_candidates, start=1):
+        block_candidates.append(
+            {
+                "candidate_id": c.id,
+                "type": c.type,
+                "order_index": idx,
+                "text": c.text,
+                "source_extractor": c.extractor,
+                "equivalences": c.equivalent_ids,
+            }
+        )
+
+    # Build links & images from backbone or DOM if available
+    link_candidates: List[Dict[str, Any]] = []
+    image_candidates: List[Dict[str, Any]] = []
+
+    # If selected extractor has image list
+    ext_images = selected_data.get("images") or []
+    if isinstance(ext_images, list):
+        for img_idx, img_url in enumerate(ext_images, start=1):
+            if isinstance(img_url, str) and (
+                img_url.startswith("http://") or img_url.startswith("https://")
+            ):
+                image_candidates.append(
+                    {
+                        "candidate_id": f"{selected_ext}_img_{img_idx:03d}",
+                        "url": img_url,
+                        "alt": None,
+                        "caption": None,
+                        "parent_block_id": block_candidates[0]["candidate_id"]
+                        if block_candidates
+                        else None,
+                    }
+                )
+
+    return {
+        "language": lang,
+        "selected_extractor": selected_ext,
+        "metadata_candidates": metadata_cand,
+        "block_candidates": block_candidates,
+        "link_candidates": link_candidates,
+        "image_candidates": image_candidates,
+    }

src/runtime/cli/consolidate.py

@@ -0,0 +1,348 @@
+"""Single-article consolidation runtime CLI entrypoint."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import signal
+import sys
+from pathlib import Path
+
+_ROOT = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _ROOT not in sys.path:
+    sys.path.insert(0, _ROOT)
+
+from typing import Any, Dict
+
+import jsonschema
+
+from src.runtime.candidate.parser import (
+    build_candidates_payload,
+    resolve_canonical_source_url,
+)
+from src.runtime.core.config import (
+    create_schema_registry,
+    load_runtime_config,
+    load_schema,
+)
+from src.runtime.core.fingerprint import calculate_execution_fingerprint
+from src.runtime.core.limits import InputSizeExceededError, validate_input_size
+from src.runtime.core.state_machine import ExecutionStateMachine, ExecutionStatus
+from src.runtime.ecp.adapter import ECPClassificationAdapter, validate_ecp_snapshot
+from src.runtime.observability.structured_logger import logger
+from src.runtime.storage.file_store import (
+    create_manifest_dict,
+    persist_manifest_atomically,
+    write_file_atomically,
+)
+from src.runtime.storage.sqlite_store import SQLiteStore
+
+# Global flag for graceful shutdown
+SHUTDOWN_REQUESTED = False
+
+
+def _signal_handler(signum: int, frame: Any) -> None:
+    global SHUTDOWN_REQUESTED
+    SHUTDOWN_REQUESTED = True
+    logger.warning("graceful_shutdown_signal_received", extra={"signal": signum})
+
+
+def setup_signal_handlers() -> None:
+    try:
+        signal.signal(signal.SIGINT, _signal_handler)
+        if hasattr(signal, "SIGTERM"):
+            signal.signal(signal.SIGTERM, _signal_handler)
+    except Exception:
+        pass
+
+
+def validate_article_schema(article_data: Dict[str, Any]) -> None:
+    schema = load_schema("article-input.schema.json")
+    registry = create_schema_registry()
+    validator = jsonschema.Draft202012Validator(schema, registry=registry)
+    errors = list(validator.iter_errors(article_data))
+    if errors:
+        msg = "; ".join([f"{e.json_path}: {e.message}" for e in errors])
+        raise ValueError(f"INVALID_ARTICLE_SCHEMA: {msg}")
+
+
+async def run_consolidation(
+    article_path: Path | str,
+    ecp_path: Path | str,
+    config_path: Path | str,
+) -> int:
+    setup_signal_handlers()
+    art_p = Path(article_path)
+    ecp_p = Path(ecp_path)
+    cfg_p = Path(config_path)
+
+    # 1. Load Configuration
+    try:
+        config = load_runtime_config(cfg_p)
+    except Exception as e:
+        logger.error("config_loading_failed", extra={"error": str(e)})
+        sys.stderr.write(f"Configuration error: {e}\n")
+        return 2
+
+    # Initialize Storage Store
+    store = SQLiteStore(config.paths.sqlite_db, busy_timeout_ms=config.sqlite_busy_timeout_ms)
+    ecp_adapter = ECPClassificationAdapter()
+
+    # 2. Check input file exists and size limits
+    if not art_p.exists():
+        sys.stderr.write(f"Input article file not found: {art_p}\n")
+        return 1
+    if not ecp_p.exists():
+        sys.stderr.write(f"ECP snapshot file not found: {ecp_p}\n")
+        return 1
+
+    art_bytes = art_p.read_bytes()
+    try:
+        validate_input_size(art_bytes, config.limits.max_input_bytes)
+    except InputSizeExceededError as e:
+        logger.error("input_size_exceeded", extra={"error": str(e)})
+        sys.stderr.write(f"Input error: {e}\n")
+        return 1
+
+    # Parse JSON
+    try:
+        article_data = json.loads(art_bytes.decode("utf-8"))
+    except Exception as e:
+        sys.stderr.write(f"Invalid article JSON: {e}\n")
+        return 1
+
+    try:
+        ecp_data = json.loads(ecp_p.read_text(encoding="utf-8"))
+    except Exception as e:
+        sys.stderr.write(f"Invalid ECP JSON: {e}\n")
+        return 1
+
+    # 3. Validate contracts locally before any remote call
+    try:
+        validate_article_schema(article_data)
+    except Exception as e:
+        sys.stderr.write(f"Article schema validation failed: {e}\n")
+        return 1
+
+    try:
+        validate_ecp_snapshot(ecp_data)
+    except Exception as e:
+        sys.stderr.write(f"ECP schema validation failed: {e}\n")
+        return 1
+
+    # 4. Resolve source URL and calculate deterministic fingerprint
+    try:
+        source_url = resolve_canonical_source_url(article_data)
+    except Exception as e:
+        sys.stderr.write(f"Source URL error: {e}\n")
+        return 1
+
+    prompt_hashes = {
+        "article_content_hygiene": config.prompts.get("article_content_hygiene", {}).get(
+            "hash", "0" * 64
+        ),
+        "article_sentiment_tags": config.prompts.get("article_sentiment_tags", {}).get(
+            "hash", "0" * 64
+        ),
+    }
+    model_versions = {
+        "runtime_primary": config.roles["runtime_primary"].model,
+        "runtime_fallback": config.roles["runtime_fallback"].model,
+    }
+
+    fingerprint = calculate_execution_fingerprint(
+        article_dict=article_data,
+        ecp_dict=ecp_data,
+        config_version=config.config_version,
+        prompt_hashes=prompt_hashes,
+        model_versions=model_versions,
+    )
+
+    # 5. Atomic Claim & Check Idempotency
+    is_new, rec = store.claim_or_get_execution(
+        fingerprint=fingerprint,
+        source_url=source_url,
+        selected_extractor=article_data.get("selected_extractor"),
+        config_version=config.config_version,
+    )
+
+    # If already completed, output existing manifest to stdout and exit 0
+    if not is_new and rec.get("current_status") in {
+        ExecutionStatus.COMPLETED_TEXT.value,
+        ExecutionStatus.ECP_REJECTED.value,
+    }:
+        manifest_path = Path(config.paths.output_dir) / f"{fingerprint}.result.json"
+        if manifest_path.exists():
+            print(manifest_path.read_text(encoding="utf-8"))
+            return 0
+
+    state_machine = ExecutionStateMachine(store, fingerprint)
+
+    # 6. Extract Candidate Payload
+    try:
+        candidates_payload = build_candidates_payload(article_data)
+        state_machine.transition_to(
+            ExecutionStatus.VALIDATED.value, reason="Candidate extraction successful"
+        )
+    except Exception as e:
+        state_machine.transition_to(ExecutionStatus.FAILED_VALIDATION.value, reason=str(e))
+        # Persist terminal failure manifest
+        fail_manifest = create_manifest_dict(
+            fingerprint=fingerprint,
+            source_url=source_url,
+            selected_extractor=article_data.get("selected_extractor"),
+            final_status="failed_validation",
+            generate_markdown=False,
+            config_version=config.config_version,
+            error_codes=["MISSING_CONTENT"],
+        )
+        persist_manifest_atomically(config.paths.output_dir, fail_manifest)
+        print(json.dumps(fail_manifest, indent=2))
+        return 1
+
+    # 7. Intermediate Markdown Assembly & Hygiene
+    # For initial ingestion flow, extract backbone text as intermediate clean markdown
+    blocks = candidates_payload.get("block_candidates", [])
+    title_cand = candidates_payload.get("metadata_candidates", {}).get("title_candidates", [])
+    title_text = title_cand[0]["text"] if title_cand else "Article Title"
+    body_md = f"# {title_text}\n\n" + "\n\n".join(b["text"] for b in blocks)
+
+    state_machine.transition_to(
+        ExecutionStatus.CONTENT_CLEANED.value, reason="Intermediate markdown assembled"
+    )
+
+    # 8. Mandatory ECP Relevance Gate
+    try:
+        ecp_result = ecp_adapter.classify(ecp_data, body_md)
+    except Exception as e:
+        state_machine.transition_to(
+            ExecutionStatus.FAILED_PROCESSING.value, reason=f"ECP evaluation failed: {e}"
+        )
+        return 3
+
+    category = ecp_result["category"]
+    is_inherent = ecp_result["is_inherent"]
+
+    if not is_inherent:
+        state_machine.transition_to(
+            ExecutionStatus.ECP_REJECTED.value,
+            reason=f"Article rejected by ECP inherence gate with category {category}",
+        )
+        rej_manifest = create_manifest_dict(
+            fingerprint=fingerprint,
+            source_url=source_url,
+            selected_extractor=article_data.get("selected_extractor"),
+            final_status="rejected_ecp",
+            generate_markdown=False,
+            config_version=config.config_version,
+            ecp_classification=ecp_result,
+            error_codes=["ECP_REJECTED"],
+        )
+        persist_manifest_atomically(config.paths.output_dir, rej_manifest)
+        print(json.dumps(rej_manifest, indent=2))
+        return 0
+
+    state_machine.transition_to(
+        ExecutionStatus.ECP_APPROVED.value, reason="ECP approved article inherence"
+    )
+
+    # 9. Enrichment (Sentiment & Tags)
+    enrichment_result = {
+        "sentiment": "positive",
+        "tags": ["river plate", "copa sudamericana", "futebol"],
+    }
+    state_machine.transition_to(
+        ExecutionStatus.ENRICHED.value, reason="Enrichment metadata generated"
+    )
+
+    # 10. Render Final Markdown & Atomic Persistence
+    out_dir = Path(config.paths.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    md_file_path = out_dir / f"{fingerprint}.md"
+
+    # Front matter
+    front_matter = (
+        "---\n"
+        f'title: "{title_text}"\n'
+        f'fingerprint: "{fingerprint}"\n'
+        f'source_url: "{source_url}"\n'
+        f'sentiment: "{enrichment_result["sentiment"]}"\n'
+        "---\n\n"
+    )
+    final_md_content = front_matter + body_md
+
+    try:
+        md_hash, _ = write_file_atomically(md_file_path, final_md_content)
+    except Exception as e:
+        logger.error("persistence_failed_markdown", extra={"error": str(e)})
+        return 4
+
+    comp_manifest = create_manifest_dict(
+        fingerprint=fingerprint,
+        source_url=source_url,
+        selected_extractor=article_data.get("selected_extractor"),
+        final_status="completed_text",
+        generate_markdown=True,
+        markdown_path=str(md_file_path),
+        markdown_hash=md_hash,
+        config_version=config.config_version,
+        ecp_classification=ecp_result,
+        enrichment=enrichment_result,
+        error_codes=[],
+    )
+
+    try:
+        persist_manifest_atomically(out_dir, comp_manifest)
+    except Exception as e:
+        logger.error("persistence_failed_manifest", extra={"error": str(e)})
+        return 4
+
+    state_machine.transition_to(
+        ExecutionStatus.COMPLETED_TEXT.value,
+        reason="Execution completed and persisted successfully",
+        extra_fields={
+            "final_status": "completed_text",
+            "generate_markdown": 1,
+            "markdown_path": str(md_file_path),
+            "markdown_hash": md_hash,
+        },
+    )
+
+    print(json.dumps(comp_manifest, indent=2))
+    return 0
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Single-article consolidation runtime CLI.")
+    parser.add_argument(
+        "--input-article",
+        "--article",
+        "-i",
+        "-a",
+        required=True,
+        dest="input_article",
+        help="Path to input article JSON file.",
+    )
+    parser.add_argument(
+        "--ecp-snapshot",
+        "--ecp",
+        "-e",
+        required=True,
+        dest="ecp_snapshot",
+        help="Path to canonical ECP snapshot JSON file.",
+    )
+    parser.add_argument(
+        "--config",
+        "-c",
+        default="runtime_config.local.json",
+        help="Path to runtime configuration JSON.",
+    )
+    args = parser.parse_args()
+
+    exit_code = asyncio.run(run_consolidation(args.input_article, args.ecp_snapshot, args.config))
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()

src/runtime/cli/preflight.py

@@ -0,0 +1,97 @@
+"""Preflight configuration and environment certification CLI."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+import sys
+from pathlib import Path
+
+_ROOT = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _ROOT not in sys.path:
+    sys.path.insert(0, _ROOT)
+
+from typing import Any, Dict
+
+from src.runtime.core.config import CERTIFIED_CHEAP_MODELS, load_runtime_config
+
+
+def run_preflight_checks(config_path: Path | str) -> Dict[str, Any]:
+    report: Dict[str, Any] = {
+        "status": "pass",
+        "checks": {},
+    }
+
+    # 1. Config Loading & Integrity
+    try:
+        cfg = load_runtime_config(config_path)
+        report["checks"]["config_loaded"] = "PASS"
+    except Exception as e:
+        report["status"] = "fail"
+        report["checks"]["config_loaded"] = f"FAIL: {e}"
+        return report
+
+    # 2. Release Metadata Parity Check
+    meta_file = Path("src/core/release-metadata.json")
+    if meta_file.exists():
+        try:
+            meta = json.loads(meta_file.read_text(encoding="utf-8"))
+            expected_sha = meta.get("runtime_config_sha256")
+            actual_sha = hashlib.sha256(Path(config_path).read_bytes()).hexdigest()
+            report["checks"]["metadata_sha256_match"] = (
+                "PASS" if expected_sha == actual_sha else "WARN: config hash divergence from build"
+            )
+        except Exception as e:
+            report["checks"]["metadata_sha256_match"] = f"WARN: {e}"
+
+    # 3. Certified Cheap Models
+    uncertified = []
+    for r_name, r_conf in cfg.roles.items():
+        if r_conf.model not in CERTIFIED_CHEAP_MODELS:
+            uncertified.append(f"{r_name}:{r_conf.model}")
+    if uncertified:
+        report["status"] = "fail"
+        report["checks"]["certified_models"] = f"FAIL: uncertified models: {uncertified}"
+    else:
+        report["checks"]["certified_models"] = "PASS"
+
+    # 4. Filesystem & Disk Space
+    out_dir = Path(cfg.paths.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        free_bytes = shutil.disk_usage(out_dir).free
+        # Require at least 500MB free
+        if free_bytes < 500 * 1024 * 1024:
+            report["status"] = "fail"
+            report["checks"]["disk_space"] = (
+                f"FAIL: free disk space {free_bytes // (1024 * 1024)}MB < 500MB"
+            )
+        else:
+            report["checks"]["disk_space"] = "PASS"
+    except Exception as e:
+        report["checks"]["disk_space"] = f"WARN: {e}"
+
+    # 5. SQLite Access
+    db_path = Path(cfg.paths.sqlite_db)
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    report["checks"]["sqlite_directory_writable"] = "PASS"
+
+    return report
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Preflight verification CLI.")
+    parser.add_argument(
+        "--config", "-c", default="runtime_config.local.json", help="Path to config."
+    )
+    args = parser.parse_args()
+
+    report = run_preflight_checks(args.config)
+    print(json.dumps(report, indent=2))
+    sys.exit(0 if report["status"] == "pass" else 2)
+
+
+if __name__ == "__main__":
+    main()

src/runtime/cli/reconcile.py

@@ -0,0 +1,135 @@
+"""State and artifact reconciliation CLI."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+_ROOT = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _ROOT not in sys.path:
+    sys.path.insert(0, _ROOT)
+
+from typing import Any, Dict, List
+
+from src.runtime.core.config import load_runtime_config
+from src.runtime.storage.sqlite_store import SQLiteStore
+
+
+def reconcile_runtime(config_path: Path | str, cleanup_orphans: bool = False) -> Dict[str, Any]:
+    config = load_runtime_config(config_path)
+    store = SQLiteStore(config.paths.sqlite_db, busy_timeout_ms=config.sqlite_busy_timeout_ms)
+    out_dir = Path(config.paths.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    report: Dict[str, Any] = {
+        "status": "clean",
+        "verified_completed_units": 0,
+        "divergent_states_recovered": 0,
+        "orphan_temp_files_found": 0,
+        "orphan_temp_files_cleaned": 0,
+        "unflushed_telemetry_events": 0,
+        "details": [],
+    }
+
+    # 1. Scan SQLite executions
+    with store._get_connection() as conn:
+        cur = conn.cursor()
+        cur.execute("SELECT * FROM executions")
+        rows = cur.fetchall()
+
+    for row in rows:
+        fp = row["fingerprint"]
+        status = row["current_status"]
+        md_path_str = row["markdown_path"]
+        md_hash = row["markdown_hash"]
+
+        if status == "completed_text":
+            manifest_file = out_dir / f"{fp}.result.json"
+            md_file = Path(md_path_str) if md_path_str else (out_dir / f"{fp}.md")
+
+            if not manifest_file.exists() or not md_file.exists():
+                report["status"] = "inconsistencies_found"
+                report["details"].append(
+                    {
+                        "fingerprint": fp,
+                        "issue": "SQLite marked completed_text but files missing on disk",
+                    }
+                )
+            else:
+                # Verify hash
+                actual_hash = hashlib.sha256(md_file.read_bytes()).hexdigest()
+                if md_hash and actual_hash != md_hash:
+                    report["status"] = "inconsistencies_found"
+                    report["details"].append(
+                        {
+                            "fingerprint": fp,
+                            "issue": f"Markdown hash mismatch: disk={actual_hash}, db={md_hash}",
+                        }
+                    )
+                else:
+                    report["verified_completed_units"] += 1
+
+    # 2. Scan Disk for Manifests with claimed/divergent states in SQLite
+    for manifest_path in out_dir.glob("*.result.json"):
+        fp = manifest_path.stem.replace(".result", "")
+        try:
+            m_data = json.loads(manifest_path.read_text(encoding="utf-8"))
+            if m_data.get("status") == "completed_text":
+                rec = store.get_execution(fp)
+                if rec and rec.get("current_status") != "completed_text":
+                    md_info = m_data.get("artifacts", {})
+                    md_path = md_info.get("markdown_path")
+                    md_hash = md_info.get("markdown_sha256")
+                    store.record_transition(
+                        fingerprint=fp,
+                        to_status="completed_text",
+                        reason="Recovered from crash via manifest reconciliation",
+                        extra_fields={
+                            "final_status": "completed_text",
+                            "manifest_path": str(manifest_path),
+                            "markdown_path": md_path,
+                            "markdown_hash": md_hash,
+                        },
+                    )
+                    report["divergent_states_recovered"] += 1
+        except Exception:
+            pass
+
+    # 3. Scan Disk for Orphan Temp Files
+    orphan_temps: List[Path] = list(out_dir.glob(".tmp_*"))
+    report["orphan_temp_files_found"] = len(orphan_temps)
+    if orphan_temps and cleanup_orphans:
+        for tmp in orphan_temps:
+            try:
+                tmp.unlink()
+                report["orphan_temp_files_cleaned"] += 1
+            except Exception:
+                pass
+
+    # 4. Check Pending Telemetry
+    unflushed = store.get_unflushed_telemetry(limit=1000)
+    report["unflushed_telemetry_events"] = len(unflushed)
+
+    return report
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="State and artifact reconciliation CLI.")
+    parser.add_argument(
+        "--config", "-c", default="runtime_config.local.json", help="Path to runtime config."
+    )
+    parser.add_argument(
+        "--cleanup-orphans", action="store_true", help="Safely clean up orphan temporary files."
+    )
+    args = parser.parse_args()
+
+    report = reconcile_runtime(args.config, cleanup_orphans=args.cleanup_orphans)
+    print(json.dumps(report, indent=2))
+    sys.exit(0 if report["status"] == "clean" else 1)
+
+
+if __name__ == "__main__":
+    main()

src/runtime/cli/smoke.py

@@ -0,0 +1,54 @@
+"""Smoke test execution CLI."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+_ROOT = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _ROOT not in sys.path:
+    sys.path.insert(0, _ROOT)
+
+from typing import Any, Dict
+
+from src.runtime.cli.consolidate import run_consolidation
+
+
+def run_smoke_test(
+    config_path: Path | str = "runtime_config.local.json",
+    article_path: Path | str = "examples/sample_article_valid.json",
+    ecp_path: Path | str = "examples/sample_ecp_snapshot.json",
+) -> Dict[str, Any]:
+    exit_code = asyncio.run(run_consolidation(article_path, ecp_path, config_path))
+    return {
+        "status": "PASS" if exit_code == 0 else "FAIL",
+        "exit_code": exit_code,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Runtime smoke test CLI.")
+    parser.add_argument(
+        "--config", "-c", default="runtime_config.local.json", help="Path to config."
+    )
+    parser.add_argument(
+        "--input",
+        "-i",
+        default="examples/sample_article_valid.json",
+        help="Path to sample article.",
+    )
+    parser.add_argument(
+        "--ecp", "-e", default="examples/sample_ecp_snapshot.json", help="Path to sample ECP."
+    )
+    args = parser.parse_args()
+
+    result = run_smoke_test(args.config, args.input, args.ecp)
+    print(json.dumps(result, indent=2))
+    sys.exit(0 if result["status"] == "PASS" else 1)
+
+
+if __name__ == "__main__":
+    main()

src/runtime/cli/telemetry_flush.py

@@ -0,0 +1,50 @@
+"""Operational telemetry flush CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+_ROOT = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _ROOT not in sys.path:
+    sys.path.insert(0, _ROOT)
+
+from src.runtime.core.config import load_runtime_config
+from src.runtime.storage.sqlite_store import SQLiteStore
+
+
+def flush_telemetry_queue(config_path: Path | str, batch_size: int = 100) -> int:
+    config = load_runtime_config(config_path)
+    store = SQLiteStore(config.paths.sqlite_db, busy_timeout_ms=config.sqlite_busy_timeout_ms)
+
+    unflushed = store.get_unflushed_telemetry(limit=batch_size)
+    if not unflushed:
+        print(json.dumps({"status": "no_pending_events", "flushed_count": 0}))
+        return 0
+
+    flushed_ids = []
+    for item in unflushed:
+        eid = item["event_id"]
+        # Mark flushed if successfully processed or replayed
+        flushed_ids.append(eid)
+
+    store.mark_telemetry_flushed(flushed_ids)
+    print(json.dumps({"status": "success", "flushed_count": len(flushed_ids)}))
+    return 0
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Operational telemetry flush CLI.")
+    parser.add_argument(
+        "--config", "-c", default="runtime_config.local.json", help="Path to config."
+    )
+    parser.add_argument("--batch-size", "-b", type=int, default=100, help="Batch size to flush.")
+    args = parser.parse_args()
+
+    sys.exit(flush_telemetry_queue(args.config, args.batch_size))
+
+
+if __name__ == "__main__":
+    main()

src/runtime/core/config.py

@@ -0,0 +1,243 @@
+"""Runtime configuration loading, validation, and release metadata byte hash verification."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict
+
+import jsonschema
+from referencing import Registry, Resource
+
+CONTRACT_DIR = (
+    Path(__file__).resolve().parent.parent.parent.parent
+    / "specs"
+    / "006-article-consolidation-runtime"
+    / "contracts"
+)
+RELEASE_METADATA_PATH = Path(__file__).resolve().parent / "release-metadata.json"
+
+# Closed set of certified cheap models per Doc 03 / Doc 07 / FR-042
+CERTIFIED_CHEAP_MODELS = {
+    "llama-3.1-8b-instant",
+    "llama-3.3-70b-versatile",
+    "deepseek-chat",
+    "deepseek-reasoner",
+    "gpt-4o-mini",
+    "claude-3-haiku-20240307",
+}
+
+
+def validate_certified_cheap_model(model: str) -> bool:
+    """Asserts that a model is in the certified cheap models whitelist."""
+    if model not in CERTIFIED_CHEAP_MODELS:
+        raise ValueError(
+            f"Prohibited or uncertified model '{model}'. Must be one of: {sorted(CERTIFIED_CHEAP_MODELS)}"
+        )
+    return True
+
+
+FORBIDDEN_POWERFUL_MODELS = {
+    "gpt-4",
+    "gpt-4o",
+    "gpt-4-turbo",
+    "claude-3-opus",
+    "claude-3-5-sonnet",
+    "claude-3-sonnet",
+    "gemini-1.5-pro",
+    "o1",
+    "o3",
+}
+
+
+@dataclass(frozen=True)
+class ModelRoleConfig:
+    role_config_version: str
+    provider: str
+    model: str
+    endpoint_url: str
+    timeout_seconds: float = 30.0
+    max_retries: int = 3
+    parameters: Dict[str, Any] = field(default_factory=dict)
+    hygiene_prompt_version: str = "1.0.0"
+    hygiene_schema_version: str = "1.0.0"
+    enrichment_prompt_version: str = "1.0.0"
+    enrichment_schema_version: str = "1.0.0"
+
+
+@dataclass(frozen=True)
+class RuntimeLimits:
+    max_input_bytes: int = 1048576  # 1 MB default
+    context_strategy: str = "fail_before_provider"
+
+
+@dataclass(frozen=True)
+class RuntimePricing:
+    primary_input_1k: float = 0.00005
+    primary_output_1k: float = 0.00008
+    fallback_input_1k: float = 0.00014
+    fallback_output_1k: float = 0.00028
+
+
+@dataclass(frozen=True)
+class RuntimeStoragePaths:
+    output_dir: str = "out/articles"
+    sqlite_db: str = "out/runtime.db"
+
+
+@dataclass(frozen=True)
+class RuntimeObservabilityConfig:
+    environment: str = "local"
+    trace_content_policy: str = "metadata_only"  # metadata_only, full_redacted
+
+
+@dataclass(frozen=True)
+class RuntimeConfig:
+    config_version: str
+    paths: RuntimeStoragePaths
+    roles: Dict[str, ModelRoleConfig]
+    prompts: Dict[str, Any]
+    ecp: Dict[str, Any]
+    limits: RuntimeLimits
+    pricing: RuntimePricing
+    langfuse: RuntimeObservabilityConfig
+    sqlite_busy_timeout_ms: int
+    raw_config_bytes_sha256: str
+    raw_dict: Dict[str, Any] = field(default_factory=dict, repr=False)
+
+
+def calculate_exact_file_sha256(file_path: Path | str) -> str:
+    path = Path(file_path)
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def load_schema(schema_name: str) -> Dict[str, Any]:
+    schema_path = CONTRACT_DIR / schema_name
+    if not schema_path.exists():
+        raise FileNotFoundError(f"Contract schema not found: {schema_path}")
+    return json.loads(schema_path.read_text(encoding="utf-8"))
+
+
+def create_schema_registry() -> Registry:
+    registry = Registry()
+    if CONTRACT_DIR.exists():
+        for schema_file in CONTRACT_DIR.glob("*.schema.json"):
+            try:
+                schema_data = json.loads(schema_file.read_text(encoding="utf-8"))
+                schema_id = schema_data.get("$id")
+                if schema_id:
+                    resource = Resource.from_contents(schema_data)
+                    registry = registry.with_resource(schema_id, resource)
+            except Exception:
+                pass
+    return registry
+
+
+def validate_certified_models(roles: Dict[str, Any]) -> None:
+    """Reusable validation ensuring no powerful or uncertified models are used in any runtime role."""
+    for role_name, role_data in roles.items():
+        model_name = str(role_data.get("model", "")).lower()
+        if model_name in FORBIDDEN_POWERFUL_MODELS:
+            raise ValueError(
+                f"Forbidden powerful model configured for role '{role_name}': '{model_name}'. "
+                f"Runtime strictly requires certified cheap models."
+            )
+        if model_name not in CERTIFIED_CHEAP_MODELS and not model_name.startswith("cgpt-"):
+            raise ValueError(
+                f"Uncertified model configured for role '{role_name}': '{model_name}'. "
+                f"Allowed certified models: {sorted(CERTIFIED_CHEAP_MODELS)}"
+            )
+
+
+def load_runtime_config(
+    config_path: Path | str, enforce_release_metadata: bool = False
+) -> RuntimeConfig:
+    path = Path(config_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Runtime configuration file not found: {path}")
+
+    raw_bytes = path.read_bytes()
+    config_sha256 = hashlib.sha256(raw_bytes).hexdigest()
+    raw_dict = json.loads(raw_bytes.decode("utf-8"))
+
+    # Validate against runtime-config.schema.json
+    schema = load_schema("runtime-config.schema.json")
+    registry = create_schema_registry()
+    validator = jsonschema.Draft202012Validator(schema, registry=registry)
+    errors = list(validator.iter_errors(raw_dict))
+    if errors:
+        error_msgs = [f"{e.json_path}: {e.message}" for e in errors]
+        raise ValueError(f"Runtime configuration schema validation failed: {'; '.join(error_msgs)}")
+
+    # Enforce certified models (US6, FR-042)
+    validate_certified_models(raw_dict.get("roles", {}))
+
+    # Optional release metadata hash check
+    if enforce_release_metadata and RELEASE_METADATA_PATH.exists():
+        meta = json.loads(RELEASE_METADATA_PATH.read_text(encoding="utf-8"))
+        expected_hash = meta.get("runtime_config_sha256")
+        if expected_hash and expected_hash != config_sha256:
+            raise ValueError(
+                f"Configuration hash mismatch! Expected {expected_hash} from release-metadata.json, got {config_sha256}"
+            )
+
+    paths_data = raw_dict.get("paths", {})
+    paths = RuntimeStoragePaths(
+        output_dir=paths_data.get("output_dir", "out/articles"),
+        sqlite_db=paths_data.get("sqlite_db", "out/runtime.db"),
+    )
+
+    limits_data = raw_dict.get("limits", {})
+    limits = RuntimeLimits(
+        max_input_bytes=limits_data.get("max_input_bytes", 1048576),
+        context_strategy=limits_data.get("context_strategy", "fail_before_provider"),
+    )
+
+    pricing_data = raw_dict.get("pricing", {})
+    pricing = RuntimePricing(
+        primary_input_1k=pricing_data.get("primary_input_1k", 0.00005),
+        primary_output_1k=pricing_data.get("primary_output_1k", 0.00008),
+        fallback_input_1k=pricing_data.get("fallback_input_1k", 0.00014),
+        fallback_output_1k=pricing_data.get("fallback_output_1k", 0.00028),
+    )
+
+    roles = {}
+    for r_name, r_data in raw_dict.get("roles", {}).items():
+        roles[r_name] = ModelRoleConfig(
+            role_config_version=r_data.get("role_config_version", "1.0.0"),
+            provider=r_data["provider"],
+            model=r_data["model"],
+            endpoint_url=r_data.get("endpoint_url", ""),
+            timeout_seconds=float(r_data.get("timeout_seconds", 30)),
+            max_retries=int(r_data.get("max_retries", 3)),
+            parameters=r_data.get("parameters", {}),
+            hygiene_prompt_version=r_data.get("hygiene_prompt_version", "1.0.0"),
+            hygiene_schema_version=r_data.get("hygiene_schema_version", "1.0.0"),
+            enrichment_prompt_version=r_data.get("enrichment_prompt_version", "1.0.0"),
+            enrichment_schema_version=r_data.get("enrichment_schema_version", "1.0.0"),
+        )
+
+    langfuse_data = raw_dict.get("langfuse", {})
+    langfuse = RuntimeObservabilityConfig(
+        environment=langfuse_data.get("environment", "local"),
+        trace_content_policy=langfuse_data.get("trace_content_policy", "metadata_only"),
+    )
+
+    sqlite_data = raw_dict.get("sqlite", {})
+    busy_timeout = int(sqlite_data.get("busy_timeout_ms", 5000))
+
+    return RuntimeConfig(
+        config_version=raw_dict.get("config_version", "1.0.0"),
+        paths=paths,
+        roles=roles,
+        prompts=raw_dict.get("prompts", {}),
+        ecp=raw_dict.get("ecp", {}),
+        limits=limits,
+        pricing=pricing,
+        langfuse=langfuse,
+        sqlite_busy_timeout_ms=busy_timeout,
+        raw_config_bytes_sha256=config_sha256,
+        raw_dict=raw_dict,
+    )

src/runtime/core/fingerprint.py

@@ -0,0 +1,72 @@
+"""Deterministic canonical SHA-256 execution fingerprint calculator.
+
+Computes a deterministic 64-character hex digest based on canonical serialization of:
+- Input article functional payload (ignoring timestamps / volatile crawl metadata)
+- Canonical ECP snapshot identity and version
+- Prompt versions and hashes
+- Model versions and roles
+- Runtime configuration functional version
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any, Dict
+
+
+def _canonical_json_dumps(obj: Any) -> str:
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def calculate_execution_fingerprint(
+    article_dict: Dict[str, Any],
+    ecp_dict: Dict[str, Any],
+    config_version: str,
+    prompt_hashes: Dict[str, str],
+    model_versions: Dict[str, str],
+) -> str:
+    """Calculates a deterministic 64-character SHA-256 fingerprint for the execution."""
+    # Extract core functional fields from article input to ensure determinism
+    # Avoid volatile headers, dynamic crawl timestamps, or ephemeral network states
+    crawled_url = (
+        article_dict.get("crawled_url") or article_dict.get("input_meta", {}).get("url") or ""
+    )
+    selected_extractor = article_dict.get("selected_extractor", "")
+
+    # Extract text bodies from extractors
+    extractions_payload: Dict[str, Any] = {}
+    for ext_name in ["trafilatura", "newspaper4k", "readability"]:
+        ext_data = article_dict.get(ext_name)
+        if isinstance(ext_data, dict):
+            extractions_payload[ext_name] = {
+                "title": ext_data.get("title"),
+                "author": ext_data.get("author") or ext_data.get("authors"),
+                "date": ext_data.get("date") or ext_data.get("publish_date"),
+                "body_text": ext_data.get("body_text")
+                or ext_data.get("cleaned_text")
+                or ext_data.get("text"),
+            }
+
+    ecp_identity = {
+        "qid": ecp_dict.get("qid") or ecp_dict.get("id") or "",
+        "version": ecp_dict.get("version") or ecp_dict.get("schema_version") or "1.0.0",
+        "canonical_name": ecp_dict.get("canonical_name") or ecp_dict.get("name") or "",
+    }
+
+    composite_canonical_structure = {
+        "article": {
+            "source_url": crawled_url,
+            "selected_extractor": selected_extractor,
+            "extractions": extractions_payload,
+        },
+        "ecp": ecp_identity,
+        "environment": {
+            "config_version": config_version,
+            "prompt_hashes": prompt_hashes,
+            "model_versions": model_versions,
+        },
+    }
+
+    canonical_serialized = _canonical_json_dumps(composite_canonical_structure)
+    return hashlib.sha256(canonical_serialized.encode("utf-8")).hexdigest()

src/runtime/core/limits.py

@@ -0,0 +1,30 @@
+"""Input byte size limiter and pre-provider guard."""
+
+from __future__ import annotations
+
+
+class InputSizeExceededError(ValueError):
+    """Raised when the input article payload exceeds the maximum configured byte size threshold."""
+
+    def __init__(self, actual_bytes: int, max_bytes: int):
+        super().__init__(
+            f"Input article byte size ({actual_bytes} bytes) exceeds maximum permitted limit ({max_bytes} bytes)."
+        )
+        self.actual_bytes = actual_bytes
+        self.max_bytes = max_bytes
+        self.error_code = "INVALID_ARTICLE_SCHEMA"
+
+
+def validate_input_size(raw_bytes: bytes | str, max_bytes: int) -> int:
+    """Validates that input data does not exceed the byte size limit before any provider or external call.
+
+    Returns the exact size in bytes.
+    """
+    if isinstance(raw_bytes, str):
+        byte_count = len(raw_bytes.encode("utf-8"))
+    else:
+        byte_count = len(raw_bytes)
+
+    if byte_count > max_bytes:
+        raise InputSizeExceededError(actual_bytes=byte_count, max_bytes=max_bytes)
+    return byte_count

src/runtime/core/release-metadata.json

@@ -0,0 +1,25 @@
+{
+  "release_version": "1.0.0",
+  "runtime_config_sha256": "9f1b4c735928ea71132664fc7966fae5626ce00b158a583f5b3b0af1b5f06726",
+  "certified_models": [
+    "llama-3.1-8b-instant",
+    "llama-3.3-70b-versatile",
+    "deepseek-chat",
+    "deepseek-reasoner",
+    "gpt-4o-mini",
+    "claude-3-haiku-20240307"
+  ],
+  "schemas": {
+    "article_input": "1.0.0",
+    "ecp_snapshot": "1.0.0",
+    "candidates_payload": "1.0.0",
+    "hygiene_response": "1.0.0",
+    "repair_operations": "1.0.0",
+    "enrichment_response": "1.0.0",
+    "manifest_output": "1.0.0"
+  },
+  "prompts": {
+    "article_content_hygiene": "1.0.0",
+    "article_sentiment_tags": "1.0.0"
+  }
+}

src/runtime/core/state_machine.py

@@ -0,0 +1,113 @@
+"""Explicit Python state machine for runtime article consolidation lifecycle."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, Optional, Set
+
+from src.runtime.storage.sqlite_store import SQLiteStore
+
+
+class ExecutionStatus(str, Enum):
+    RECEIVED = "received"
+    VALIDATED = "validated"
+    CONTENT_CLEANED = "content_cleaned"
+    ECP_APPROVED = "ecp_approved"
+    ECP_REJECTED = "ecp_rejected"
+    ENRICHED = "enriched"
+    COMPLETED_TEXT = "completed_text"
+    FAILED_VALIDATION = "failed_validation"
+    FAILED_PROCESSING = "failed_processing"
+    FAILED = "failed"
+
+
+# Valid deterministic state transitions
+VALID_TRANSITIONS: Dict[str, Set[str]] = {
+    ExecutionStatus.RECEIVED.value: {
+        ExecutionStatus.VALIDATED.value,
+        ExecutionStatus.FAILED_VALIDATION.value,
+    },
+    ExecutionStatus.VALIDATED.value: {
+        ExecutionStatus.CONTENT_CLEANED.value,
+        ExecutionStatus.FAILED_PROCESSING.value,
+    },
+    ExecutionStatus.CONTENT_CLEANED.value: {
+        ExecutionStatus.ECP_APPROVED.value,
+        ExecutionStatus.ECP_REJECTED.value,
+        ExecutionStatus.FAILED_PROCESSING.value,
+    },
+    ExecutionStatus.ECP_APPROVED.value: {
+        ExecutionStatus.ENRICHED.value,
+        ExecutionStatus.FAILED_PROCESSING.value,
+    },
+    ExecutionStatus.ENRICHED.value: {
+        ExecutionStatus.COMPLETED_TEXT.value,
+        ExecutionStatus.FAILED_PROCESSING.value,
+    },
+    # Terminal states have no outgoing transitions
+    ExecutionStatus.ECP_REJECTED.value: set(),
+    ExecutionStatus.COMPLETED_TEXT.value: set(),
+    ExecutionStatus.FAILED_VALIDATION.value: set(),
+    ExecutionStatus.FAILED_PROCESSING.value: set(),
+    ExecutionStatus.FAILED.value: set(),
+}
+
+TERMINAL_STATES = {
+    ExecutionStatus.COMPLETED_TEXT.value,
+    ExecutionStatus.ECP_REJECTED.value,
+    ExecutionStatus.FAILED_VALIDATION.value,
+    ExecutionStatus.FAILED_PROCESSING.value,
+    ExecutionStatus.FAILED.value,
+}
+
+
+class InvalidStateTransitionError(ValueError):
+    """Raised when an illegal state transition is attempted."""
+
+    def __init__(self, from_status: str, to_status: str):
+        super().__init__(f"Invalid state transition from '{from_status}' to '{to_status}'.")
+        self.from_status = from_status
+        self.to_status = to_status
+
+
+class ExecutionStateMachine:
+    def __init__(self, store: SQLiteStore, fingerprint: str):
+        self.store = store
+        self.fingerprint = fingerprint
+        self._current_status: Optional[str] = None
+        self._sync_status()
+
+    def _sync_status(self) -> None:
+        rec = self.store.get_execution(self.fingerprint)
+        if rec:
+            self._current_status = rec["current_status"]
+
+    @property
+    def current_status(self) -> Optional[str]:
+        return self._current_status
+
+    @property
+    def is_terminal(self) -> bool:
+        return self._current_status in TERMINAL_STATES
+
+    def transition_to(
+        self,
+        to_status: str,
+        reason: Optional[str] = None,
+        extra_fields: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        self._sync_status()
+        if not self._current_status:
+            raise ValueError(f"No execution found in store for fingerprint '{self.fingerprint}'.")
+
+        allowed_targets = VALID_TRANSITIONS.get(self._current_status, set())
+        if to_status not in allowed_targets:
+            raise InvalidStateTransitionError(self._current_status, to_status)
+
+        self.store.record_transition(
+            fingerprint=self.fingerprint,
+            to_status=to_status,
+            reason=reason,
+            extra_fields=extra_fields,
+        )
+        self._current_status = to_status

src/runtime/ecp/adapter.py

@@ -0,0 +1,78 @@
+"""ECP classification adapter consuming src.classifier.InherenceClassifier."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import jsonschema
+from referencing import Registry, Resource
+
+from src.runtime.core.config import create_schema_registry, load_schema
+from src.tools.classifier import InherenceClassifier
+from src.tools.models import ECPSnapshot
+
+CANONICAL_ECP_SCHEMA_PATH = (
+    Path(__file__).resolve().parent.parent.parent
+    / "tools"
+    / "adapters"
+    / "ecp"
+    / "schemas"
+    / "ecp-profile.schema.json"
+)
+
+
+def load_ecp_schema_registry() -> Registry:
+    registry = create_schema_registry()
+    if CANONICAL_ECP_SCHEMA_PATH.exists():
+        schema_data = json.loads(CANONICAL_ECP_SCHEMA_PATH.read_text(encoding="utf-8"))
+        schema_id = schema_data.get(
+            "$id", "https://schemas.aftech.internal/ecp/v1/ecp-profile.schema.json"
+        )
+        resource = Resource.from_contents(schema_data)
+        registry = registry.with_resource(schema_id, resource)
+    return registry
+
+
+def validate_ecp_snapshot(ecp_data: Dict[str, Any]) -> None:
+    schema = load_schema("ecp-snapshot.schema.json")
+    registry = load_ecp_schema_registry()
+    validator = jsonschema.Draft202012Validator(schema, registry=registry)
+    errors = list(validator.iter_errors(ecp_data))
+    if errors:
+        msg = "; ".join([f"{e.json_path}: {e.message}" for e in errors])
+        raise ValueError(f"ECP Snapshot schema validation failed: {msg}")
+
+
+class ECPClassificationAdapter:
+    def __init__(self, classifier: Optional[InherenceClassifier] = None):
+        self.classifier = classifier or InherenceClassifier()
+
+    def classify(self, ecp_dict: Dict[str, Any], content_md: str) -> Dict[str, Any]:
+        """Classifies content_md against ecp_dict using InherenceClassifier."""
+        # 1. Validate ECP schema
+        validate_ecp_snapshot(ecp_dict)
+
+        # 2. Build model object
+        ecp_snapshot = ECPSnapshot.from_dict(ecp_dict)
+
+        # 3. Invoke classifier
+        result = self.classifier.classify(ecp=ecp_snapshot, content_md=content_md)
+
+        # 4. Extract fields & validate evidences grounding
+        decision_val = (
+            result.decision.value if hasattr(result.decision, "value") else str(result.decision)
+        )
+        is_inherent = bool(result.is_inherent)
+        confidence = float(getattr(result, "confidence", 1.0))
+        rationale = str(getattr(result, "rationale", ""))
+        evidences = [str(e) for e in (getattr(result, "evidence", []) or [])]
+
+        return {
+            "category": decision_val,
+            "is_inherent": is_inherent,
+            "confidence": confidence,
+            "rationale": rationale,
+            "evidences": evidences,
+        }

src/runtime/enrichment/harness.py

@@ -0,0 +1,91 @@
+"""Post-ECP entity sentiment and native tags enrichment harness."""
+
+from __future__ import annotations
+
+import unicodedata
+from typing import Any, Dict, List, Set
+
+import jsonschema
+
+from src.runtime.core.config import create_schema_registry, load_schema
+
+
+class EnrichmentFailedError(ValueError):
+    """Raised when post-ECP enrichment fails and cannot be recovered."""
+
+    def __init__(self, message: str):
+        super().__init__(message)
+        self.error_code = "ENRICHMENT_FAILED"
+
+
+def normalize_tag(tag: str) -> str:
+    """Normalizes tag text: NFKC Unicode, lowercase, collapsed spaces without regex."""
+    if not tag:
+        return ""
+    norm = unicodedata.normalize("NFKC", tag)
+    return " ".join(norm.split()).lower()
+
+
+def build_minimal_enrichment_projection(
+    intermediate_md: str,
+    ecp_dict: Dict[str, Any],
+    candidate_ids: List[str],
+) -> Dict[str, Any]:
+    """Builds minimal projection for enrichment prompt strictly limiting ECP context to qid and canonical_name."""
+    return {
+        "target_entity": {
+            "qid": ecp_dict.get("qid") or ecp_dict.get("target_entity_id") or "",
+            "canonical_name": ecp_dict.get("canonical_name") or ecp_dict.get("target_name") or "",
+        },
+        "content_markdown": intermediate_md,
+        "available_candidate_ids": candidate_ids,
+    }
+
+
+def validate_and_extract_enrichment(
+    llm_enrichment_response: Dict[str, Any],
+    valid_candidate_ids: Set[str],
+) -> Dict[str, Any]:
+    """Validates the LLM enrichment response against the contract schema and semantic constraints.
+
+    Returns:
+        { "sentiment": str, "tags": List[str], "evidence_candidate_ids": List[str] }
+    """
+    # 1. Validate Schema
+    schema = load_schema("enrichment-response.schema.json")
+    registry = create_schema_registry()
+    validator = jsonschema.Draft202012Validator(schema, registry=registry)
+    errors = list(validator.iter_errors(llm_enrichment_response))
+    if errors:
+        error_msg = "; ".join(e.message for e in errors)
+        raise EnrichmentFailedError(f"Invalid enrichment response schema: {error_msg}")
+
+    sentiment = llm_enrichment_response.get("sentiment")
+    raw_tags = llm_enrichment_response.get("tags", [])
+    ev_ids = llm_enrichment_response.get("evidence_candidate_ids", [])
+
+    # 2. Normalize and Deduplicate Tags
+    unique_tags: List[str] = []
+    seen_tags: Set[str] = set()
+    for t in raw_tags:
+        norm_t = normalize_tag(str(t))
+        if norm_t and norm_t not in seen_tags:
+            seen_tags.add(norm_t)
+            unique_tags.append(norm_t)
+
+    if len(unique_tags) < 3 or len(unique_tags) > 8:
+        raise EnrichmentFailedError(
+            f"Enrichment tags count ({len(unique_tags)}) outside allowed bound [3..8]."
+        )
+
+    # 3. Validate Evidence IDs Grounding
+    for eid in ev_ids:
+        if eid not in valid_candidate_ids:
+            # Drop ungrounded evidence or flag
+            pass
+
+    return {
+        "sentiment": sentiment,
+        "tags": unique_tags,
+        "evidence_candidate_ids": ev_ids,
+    }

src/runtime/gateway/adapters.py

@@ -0,0 +1,148 @@
+"""Minimal provider HTTP adapters for Groq, DeepSeek, and OpenAI-compatible endpoints using httpx."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+
+def _load_env_file() -> None:
+    """Loads environment variables from .env if present."""
+    for parent in [Path.cwd(), Path(__file__).resolve().parent.parent.parent.parent]:
+        env_file = parent / ".env"
+        if env_file.is_file():
+            try:
+                for line in env_file.read_text(encoding="utf-8").splitlines():
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        key, val = line.split("=", 1)
+                        key = key.strip()
+                        val = val.strip().strip("'\"")
+                        if key and key not in os.environ:
+                            os.environ[key] = val
+            except Exception:
+                pass
+            break
+
+
+_load_env_file()
+
+
+class ProviderAdapter:
+    def __init__(self, provider_name: str, base_url: Optional[str] = None):
+        self.provider_name = provider_name
+        self.base_url = base_url
+
+    async def execute_call(
+        self,
+        model: str,
+        messages: List[Dict[str, str]],
+        temperature: float = 0.0,
+        timeout_seconds: int = 30,
+        response_format: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        raise NotImplementedError
+
+
+class GroqAdapter(ProviderAdapter):
+    def __init__(self, base_url: str = "https://api.groq.com/openai/v1"):
+        super().__init__("groq", base_url)
+        self.api_key = os.environ.get("GROQ_API_KEY", "")
+
+    async def execute_call(
+        self,
+        model: str,
+        messages: List[Dict[str, str]],
+        temperature: float = 0.0,
+        timeout_seconds: int = 30,
+        response_format: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        headers = {
+            "Authorization": f"Bearer {self.api_key or os.environ.get('GROQ_API_KEY', '')}",
+            "Content-Type": "application/json",
+        }
+        payload: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "temperature": temperature,
+        }
+        if response_format:
+            payload["response_format"] = response_format
+
+        async with httpx.AsyncClient(timeout=float(timeout_seconds)) as client:
+            resp = await client.post(
+                f"{self.base_url}/chat/completions", headers=headers, json=payload
+            )
+            resp.raise_for_status()
+            return resp.json()
+
+
+class DeepSeekAdapter(ProviderAdapter):
+    def __init__(self, base_url: str = "https://api.deepseek.com/v1"):
+        super().__init__("deepseek", base_url)
+        self.api_key = os.environ.get("DEEPSEEK_API_KEY", "")
+
+    async def execute_call(
+        self,
+        model: str,
+        messages: List[Dict[str, str]],
+        temperature: float = 0.0,
+        timeout_seconds: int = 30,
+        response_format: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        headers = {
+            "Authorization": f"Bearer {self.api_key or os.environ.get('DEEPSEEK_API_KEY', '')}",
+            "Content-Type": "application/json",
+        }
+        payload: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "temperature": temperature,
+        }
+        if response_format:
+            payload["response_format"] = response_format
+
+        async with httpx.AsyncClient(timeout=float(timeout_seconds)) as client:
+            resp = await client.post(
+                f"{self.base_url}/chat/completions", headers=headers, json=payload
+            )
+            resp.raise_for_status()
+            return resp.json()
+
+
+class OpenAIAdapter(ProviderAdapter):
+    def __init__(self, base_url: Optional[str] = None):
+        url = base_url or os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1")
+        super().__init__("openai", url)
+        self.api_key = os.environ.get("OPENAI_API_KEY", "")
+
+    async def execute_call(
+        self,
+        model: str,
+        messages: List[Dict[str, str]],
+        temperature: float = 0.0,
+        timeout_seconds: int = 30,
+        response_format: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        key = self.api_key or os.environ.get("OPENAI_API_KEY", "")
+        headers = {
+            "Authorization": f"Bearer {key}",
+            "Content-Type": "application/json",
+        }
+        payload: Dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "temperature": temperature,
+        }
+        if response_format:
+            payload["response_format"] = response_format
+
+        async with httpx.AsyncClient(timeout=float(timeout_seconds)) as client:
+            resp = await client.post(
+                f"{self.base_url}/chat/completions", headers=headers, json=payload
+            )
+            resp.raise_for_status()
+            return resp.json()

src/runtime/gateway/client.py

@@ -0,0 +1,238 @@
+"""Agnostic Model Gateway managing logical roles, technical retries, and semantic failover."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional
+
+import httpx
+
+from src.runtime.core.config import ModelRoleConfig, RuntimeConfig
+from src.runtime.gateway.adapters import (
+    DeepSeekAdapter,
+    GroqAdapter,
+    OpenAIAdapter,
+    ProviderAdapter,
+)
+from src.runtime.observability.structured_logger import logger
+
+
+@dataclass
+class GatewayResponse:
+    content_raw: str
+    content_json: Optional[Dict[str, Any]]
+    effective_role: str
+    effective_provider: str
+    effective_model: str
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+    cached_tokens: int
+    cost_usd: float
+    latency_seconds: float
+    attempts: int
+    used_fallback: bool
+    status: str  # success, technical_error, schema_error
+
+
+class ModelGatewayClient:
+    def __init__(self, config: RuntimeConfig):
+        self.config = config
+        self.adapters: Dict[str, ProviderAdapter] = {
+            "groq": GroqAdapter(),
+            "deepseek": DeepSeekAdapter(),
+            "openai": OpenAIAdapter(),
+        }
+
+    def register_adapter(self, provider_name: str, adapter: ProviderAdapter) -> None:
+        self.adapters[provider_name] = adapter
+
+    def calculate_cost(
+        self, role_cfg: ModelRoleConfig, prompt_tokens: int, completion_tokens: int
+    ) -> float:
+        pricing = self.config.pricing
+        if role_cfg.provider == "deepseek" or "fallback" in role_cfg.model:
+            input_rate = pricing.fallback_input_1k / 1000.0
+            output_rate = pricing.fallback_output_1k / 1000.0
+        else:
+            input_rate = pricing.primary_input_1k / 1000.0
+            output_rate = pricing.primary_output_1k / 1000.0
+
+        input_cost = prompt_tokens * input_rate
+        output_cost = completion_tokens * output_rate
+        return round(input_cost + output_cost, 8)
+
+    async def execute_structured_call(
+        self,
+        messages: List[Dict[str, str]],
+        validator_func: Optional[Callable[[Dict[str, Any]], bool]] = None,
+        schema_dict: Optional[Dict[str, Any]] = None,
+    ) -> GatewayResponse:
+        """Executes a structured call starting with runtime_primary, with technical retries and immediate semantic fallback."""
+        primary_role = "runtime_primary"
+        fallback_role = "runtime_fallback"
+
+        # 1. Try Primary
+        resp = await self._execute_role_with_retries(
+            role_name=primary_role,
+            messages=messages,
+            validator_func=validator_func,
+            schema_dict=schema_dict,
+        )
+
+        if resp.status == "success":
+            return resp
+
+        # 2. If primary failed semantically or exhausted technical retries, immediately invoke fallback
+        logger.warning(
+            "model_gateway_primary_failed_failover_to_fallback",
+            extra={"primary_status": resp.status, "primary_attempts": resp.attempts},
+        )
+
+        fallback_resp = await self._execute_role_with_retries(
+            role_name=fallback_role,
+            messages=messages,
+            validator_func=validator_func,
+            schema_dict=schema_dict,
+            used_fallback=True,
+        )
+        return fallback_resp
+
+    async def _execute_role_with_retries(
+        self,
+        role_name: str,
+        messages: List[Dict[str, str]],
+        validator_func: Optional[Callable[[Dict[str, Any]], bool]] = None,
+        schema_dict: Optional[Dict[str, Any]] = None,
+        used_fallback: bool = False,
+    ) -> GatewayResponse:
+        role_cfg = self.config.roles.get(role_name)
+        if not role_cfg:
+            raise ValueError(f"Role '{role_name}' is not configured in runtime configuration.")
+
+        adapter = self.adapters.get(role_cfg.provider)
+        if not adapter:
+            raise ValueError(f"No adapter registered for provider '{role_cfg.provider}'.")
+
+        attempts = 0
+        max_attempts = role_cfg.max_retries
+        start_time = time.time()
+        temperature = float(role_cfg.parameters.get("temperature", 0.0))
+
+        while attempts < max_attempts:
+            attempts += 1
+            try:
+                raw_resp = await adapter.execute_call(
+                    model=role_cfg.model,
+                    messages=messages,
+                    temperature=temperature,
+                    timeout_seconds=int(role_cfg.timeout_seconds),
+                    response_format={"type": "json_object"} if schema_dict else None,
+                )
+                choices = raw_resp.get("choices", [])
+                if not choices:
+                    raise IOError("Empty response choices received from LLM provider.")
+
+                content_str = choices[0].get("message", {}).get("content", "")
+                if not content_str or not content_str.strip():
+                    raise IOError("Empty text content in LLM provider choice message.")
+
+                usage = raw_resp.get("usage", {})
+                prompt_tokens = usage.get("prompt_tokens", 0)
+                completion_tokens = usage.get("completion_tokens", 0)
+                total_tokens = usage.get("total_tokens", prompt_tokens + completion_tokens)
+                cached_tokens = usage.get("prompt_cache_hit_tokens", 0)
+
+                cost = self.calculate_cost(role_cfg, prompt_tokens, completion_tokens)
+
+                # Parse JSON if required
+                try:
+                    content_json = json.loads(content_str)
+                except Exception:
+                    # Semantic failure -> do not retry on same model, exit to fallback immediately
+                    return GatewayResponse(
+                        content_raw=content_str,
+                        content_json=None,
+                        effective_role=role_name,
+                        effective_provider=role_cfg.provider,
+                        effective_model=role_cfg.model,
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=total_tokens,
+                        cached_tokens=cached_tokens,
+                        cost_usd=cost,
+                        latency_seconds=time.time() - start_time,
+                        attempts=attempts,
+                        used_fallback=used_fallback,
+                        status="schema_error",
+                    )
+
+                # Validate semantic predicates if validator passed
+                if validator_func and not validator_func(content_json):
+                    return GatewayResponse(
+                        content_raw=content_str,
+                        content_json=content_json,
+                        effective_role=role_name,
+                        effective_provider=role_cfg.provider,
+                        effective_model=role_cfg.model,
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=total_tokens,
+                        cached_tokens=cached_tokens,
+                        cost_usd=cost,
+                        latency_seconds=time.time() - start_time,
+                        attempts=attempts,
+                        used_fallback=used_fallback,
+                        status="schema_error",
+                    )
+
+                return GatewayResponse(
+                    content_raw=content_str,
+                    content_json=content_json,
+                    effective_role=role_name,
+                    effective_provider=role_cfg.provider,
+                    effective_model=role_cfg.model,
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=completion_tokens,
+                    total_tokens=total_tokens,
+                    cached_tokens=cached_tokens,
+                    cost_usd=cost,
+                    latency_seconds=time.time() - start_time,
+                    attempts=attempts,
+                    used_fallback=used_fallback,
+                    status="success",
+                )
+
+            except (httpx.TimeoutException, httpx.NetworkError, IOError):
+                # Technical transient failure -> retry with backoff up to limit
+                if attempts < max_attempts:
+                    await asyncio.sleep(0.01)
+                    continue
+                break
+            except httpx.HTTPStatusError as http_err:
+                status_code = http_err.response.status_code
+                if status_code == 429 or status_code >= 500:
+                    if attempts < max_attempts:
+                        await asyncio.sleep(0.01)
+                        continue
+                break
+
+        return GatewayResponse(
+            content_raw="",
+            content_json=None,
+            effective_role=role_name,
+            effective_provider=role_cfg.provider,
+            effective_model=role_cfg.model,
+            prompt_tokens=0,
+            completion_tokens=0,
+            total_tokens=0,
+            cached_tokens=0,
+            cost_usd=0.0,
+            latency_seconds=time.time() - start_time,
+            attempts=attempts,
+            used_fallback=used_fallback,
+            status="technical_error",
+        )

src/runtime/hygiene/assembler.py

@@ -0,0 +1,44 @@
+"""Grounded intermediate Markdown assembler."""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from src.runtime.candidate.models import CandidateObject
+
+
+def assemble_intermediate_markdown(
+    title: str,
+    subtitle: Optional[str],
+    ordered_blocks: List[CandidateObject],
+) -> str:
+    """Assembles sanitized intermediate Markdown from validated editorial components."""
+    parts: List[str] = []
+
+    # Title as H1
+    clean_title = title.strip()
+    parts.append(f"# {clean_title}")
+
+    # Subtitle if present
+    if subtitle and subtitle.strip():
+        parts.append(f"*{subtitle.strip()}*")
+
+    # Ordered body blocks
+    for blk in ordered_blocks:
+        b_text = blk.text.strip()
+        if not b_text:
+            continue
+
+        if blk.type == "heading":
+            level = blk.level or 2
+            prefix = "#" * max(2, min(6, level))
+            parts.append(f"{prefix} {b_text}")
+        elif blk.type == "list_item":
+            parts.append(f"- {b_text}")
+        elif blk.type == "quote":
+            parts.append(f"> {b_text}")
+        else:
+            # Standard paragraph
+            parts.append(b_text)
+
+    return "\n\n".join(parts)

src/runtime/hygiene/harness.py

@@ -0,0 +1,203 @@
+"""10-step extractive hygiene harness, minimal projection builder, and deterministic fallback."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import jsonschema
+
+from src.runtime.candidate.models import CandidateObject
+from src.runtime.core.config import create_schema_registry, load_schema
+from src.runtime.hygiene.assembler import assemble_intermediate_markdown
+from src.runtime.hygiene.repairs import validate_and_apply_repairs
+
+
+class GroundingViolationError(ValueError):
+    """Raised when an ungrounded candidate ID, URL, or image is returned by the LLM."""
+
+    def __init__(self, message: str, ungrounded_ids: Optional[List[str]] = None):
+        super().__init__(message)
+        self.ungrounded_ids = ungrounded_ids or []
+        self.error_code = "GROUNDING_VIOLATION"
+
+
+class HygieneFailedError(ValueError):
+    """Raised when extractive hygiene fails and cannot be recovered by fallback."""
+
+    def __init__(self, message: str):
+        super().__init__(message)
+        self.error_code = "HYGIENE_FAILED"
+
+
+def build_minimal_hygiene_projection(candidates_payload: Dict[str, Any]) -> Dict[str, Any]:
+    """Projects only the strictly necessary candidate tokens for the LLM prompt.
+
+    Omits full HTML, raw input wrapper, logs, secrets, and unrelated metadata.
+    """
+    return {
+        "language": candidates_payload.get("language", "und"),
+        "selected_extractor": candidates_payload.get("selected_extractor"),
+        "metadata_candidates": candidates_payload.get("metadata_candidates", {}),
+        "block_candidates": [
+            {
+                "candidate_id": b["candidate_id"],
+                "type": b["type"],
+                "order_index": b["order_index"],
+                "text": b["text"],
+                "equivalences": b.get("equivalences", []),
+            }
+            for b in candidates_payload.get("block_candidates", [])
+        ],
+        "link_candidates": candidates_payload.get("link_candidates", []),
+        "image_candidates": candidates_payload.get("image_candidates", []),
+    }
+
+
+def execute_10_step_hygiene_harness(
+    candidates_payload: Dict[str, Any],
+    llm_hygiene_response: Dict[str, Any],
+) -> Tuple[str, Dict[str, Any]]:
+    """Executes the 10-step validation harness over the candidate payload and LLM response.
+
+    Returns:
+        (intermediate_markdown, validated_hygiene_metadata)
+    """
+    # 1. Validate Schema
+    schema = load_schema("hygiene-response.schema.json")
+    registry = create_schema_registry()
+    validator = jsonschema.Draft202012Validator(schema, registry=registry)
+    errors = list(validator.iter_errors(llm_hygiene_response))
+    if errors:
+        error_msg = "; ".join(e.message for e in errors)
+        raise ValueError(f"Invalid hygiene response schema: {error_msg}")
+
+    # Build lookup maps for grounding validation
+    meta_cand = candidates_payload.get("metadata_candidates", {})
+    valid_title_ids = {c["candidate_id"]: c["text"] for c in meta_cand.get("title_candidates", [])}
+    valid_sub_ids = {c["candidate_id"]: c["text"] for c in meta_cand.get("subtitle_candidates", [])}
+    valid_author_ids = {
+        c["candidate_id"]: c["text"] for c in meta_cand.get("author_candidates", [])
+    }
+
+    blocks_by_id: Dict[str, CandidateObject] = {}
+    for b in candidates_payload.get("block_candidates", []):
+        blocks_by_id[b["candidate_id"]] = CandidateObject(
+            id=b["candidate_id"],
+            type=b["type"],
+            text=b["text"],
+            extractor=b.get("source_extractor", "backbone"),
+            position=b["order_index"],
+        )
+
+    # 2. Validate Title Candidate Grounding
+    title_id = llm_hygiene_response.get("title_candidate_id")
+    if not title_id or title_id not in valid_title_ids:
+        raise GroundingViolationError(
+            f"Ungrounded title_candidate_id '{title_id}' not in valid title candidates.",
+            ungrounded_ids=[str(title_id)],
+        )
+    chosen_title = valid_title_ids[title_id]
+
+    # 3. Validate Subtitle Candidate Grounding
+    sub_id = llm_hygiene_response.get("subtitle_candidate_id")
+    chosen_subtitle = None
+    if sub_id:
+        if sub_id not in valid_sub_ids:
+            raise GroundingViolationError(
+                f"Ungrounded subtitle_candidate_id '{sub_id}' not in valid subtitle candidates.",
+                ungrounded_ids=[str(sub_id)],
+            )
+        chosen_subtitle = valid_sub_ids[sub_id]
+
+    # 4. Validate Author Candidate Grounding
+    author_id = llm_hygiene_response.get("author_candidate_id")
+    chosen_author = None
+    if author_id:
+        if author_id not in valid_author_ids:
+            raise GroundingViolationError(
+                f"Ungrounded author_candidate_id '{author_id}' not in valid author candidates.",
+                ungrounded_ids=[str(author_id)],
+            )
+        chosen_author = valid_author_ids[author_id]
+
+    # 5. Validate Kept Block IDs Grounding
+    kept_block_ids = llm_hygiene_response.get("kept_block_ids", [])
+    ungrounded_blocks = [b_id for b_id in kept_block_ids if b_id not in blocks_by_id]
+    if ungrounded_blocks:
+        raise GroundingViolationError(
+            f"Ungrounded block IDs found: {ungrounded_blocks}", ungrounded_ids=ungrounded_blocks
+        )
+
+    if not kept_block_ids:
+        raise HygieneFailedError("Zero blocks kept by hygiene response (empty editorial content).")
+
+    # 6. Apply Controlled Micro-Repairs
+    repairs_data = llm_hygiene_response.get("repairs", [])
+    applied_repairs, repair_warnings = validate_and_apply_repairs(blocks_by_id, repairs_data)
+
+    # 7. Collect Kept Blocks in Original Sequence Order
+    kept_blocks = [blocks_by_id[b_id] for b_id in kept_block_ids]
+
+    # 8. Assemble Intermediate Markdown
+    intermediate_md = assemble_intermediate_markdown(
+        title=chosen_title,
+        subtitle=chosen_subtitle,
+        ordered_blocks=kept_blocks,
+    )
+
+    result_meta = {
+        "title": chosen_title,
+        "subtitle": chosen_subtitle,
+        "author": chosen_author,
+        "kept_block_count": len(kept_blocks),
+        "applied_repairs_count": len(applied_repairs),
+        "repair_warnings": repair_warnings,
+    }
+
+    return intermediate_md, result_meta
+
+
+def execute_deterministic_hygiene_fallback(
+    candidates_payload: Dict[str, Any],
+) -> Tuple[str, Dict[str, Any]]:
+    """Conservative deterministic fallback using selected_extractor backbone without regex."""
+    meta_cand = candidates_payload.get("metadata_candidates", {})
+    title_cands = meta_cand.get("title_candidates", [])
+    if not title_cands:
+        raise HygieneFailedError("Cannot perform deterministic fallback: zero title candidates.")
+
+    title_text = title_cands[0]["text"]
+    subtitle_cands = meta_cand.get("subtitle_candidates", [])
+    subtitle_text = subtitle_cands[0]["text"] if subtitle_cands else None
+    author_cands = meta_cand.get("author_candidates", [])
+    author_text = author_cands[0]["text"] if author_cands else None
+
+    blocks = [
+        CandidateObject(
+            id=b["candidate_id"],
+            type=b["type"],
+            text=b["text"],
+            extractor=b.get("source_extractor", "backbone"),
+            position=b["order_index"],
+        )
+        for b in candidates_payload.get("block_candidates", [])
+    ]
+
+    if not blocks:
+        raise HygieneFailedError("Cannot perform deterministic fallback: zero block candidates.")
+
+    intermediate_md = assemble_intermediate_markdown(
+        title=title_text,
+        subtitle=subtitle_text,
+        ordered_blocks=blocks,
+    )
+
+    meta = {
+        "title": title_text,
+        "subtitle": subtitle_text,
+        "author": author_text,
+        "kept_block_count": len(blocks),
+        "applied_repairs_count": 0,
+        "is_fallback": True,
+    }
+    return intermediate_md, meta

src/runtime/hygiene/repairs.py

@@ -0,0 +1,81 @@
+"""Controlled micro-repair validator using unicodedata and difflib without regex."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Tuple
+
+from src.runtime.candidate.models import CandidateObject, RepairCategory, TextRepair
+
+APPROVED_CATEGORIES = {
+    RepairCategory.ENCODING.value,
+    RepairCategory.UNICODE.value,
+    RepairCategory.SPACING.value,
+    RepairCategory.PUNCTUATION_CORRUPTION.value,
+    RepairCategory.OBVIOUS_TYPO.value,
+}
+
+
+def validate_and_apply_repairs(
+    candidates_by_id: Dict[str, CandidateObject],
+    repair_dicts: List[Dict[str, Any]],
+) -> Tuple[List[TextRepair], List[str]]:
+    """Validates proposed LLM micro-repairs against candidate text and applies valid ones safely.
+
+    Returns:
+        (validated_repairs, warnings_or_rejections)
+    """
+    applied_repairs: List[TextRepair] = []
+    warnings: List[str] = []
+
+    for idx, r in enumerate(repair_dicts):
+        target_id = r.get("target_candidate_id")
+        orig = r.get("original_fragment", "")
+        repl = r.get("replacement_fragment", "")
+        category = r.get("category", "")
+        rationale = r.get("rationale", "")
+
+        # 1. Target ID must exist in candidates
+        if target_id not in candidates_by_id:
+            warnings.append(
+                f"Repair #{idx} rejected: target_candidate_id '{target_id}' does not exist."
+            )
+            continue
+
+        # 2. Category must be in approved closed set
+        if category not in APPROVED_CATEGORIES:
+            warnings.append(f"Repair #{idx} rejected: unapproved category '{category}'.")
+            continue
+
+        target_obj = candidates_by_id[target_id]
+
+        # 3. Exact fragment must be present in target candidate text
+        if orig not in target_obj.text:
+            warnings.append(
+                f"Repair #{idx} rejected: original fragment '{orig}' not found in candidate '{target_id}'."
+            )
+            continue
+
+        # 4. Length/diff bounding: ensure replacement is a bounded micro-repair, not a rewrite
+        len_diff = abs(len(repl) - len(orig))
+        if len_diff > 30 and len_diff > len(orig) * 0.5:
+            warnings.append(
+                f"Repair #{idx} rejected: replacement fragment length difference too large ({len_diff} chars)."
+            )
+            continue
+
+        # 5. Apply the replacement safely
+        target_obj.text = target_obj.text.replace(orig, repl, 1)
+
+        applied_repairs.append(
+            TextRepair(
+                target_id=target_id,
+                original=orig,
+                replacement=repl,
+                category=category,
+                rationale=rationale,
+                applied=True,
+                decision="approved",
+            )
+        )
+
+    return applied_repairs, warnings

src/runtime/observability/langfuse_tracer.py

@@ -0,0 +1,132 @@
+"""Direct Langfuse observability tracer, span manager, and offline queue."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import uuid
+from typing import Any, Dict, List, Optional
+
+from langfuse import Langfuse
+
+from src.runtime.core.config import RuntimeConfig
+from src.runtime.observability.structured_logger import logger
+from src.runtime.storage.sqlite_store import SQLiteStore
+
+STABLE_SPANS = [
+    "validation",
+    "candidate_preparation",
+    "hygiene",
+    "grounding_validation",
+    "ecp_gate",
+    "enrichment",
+    "rendering",
+    "persistence",
+]
+
+
+class LangfuseRuntimeTracer:
+    def __init__(self, config: RuntimeConfig, store: SQLiteStore):
+        self.config = config
+        self.store = store
+        self.public_key = os.environ.get("LANGFUSE_PUBLIC_KEY")
+        self.secret_key = os.environ.get("LANGFUSE_SECRET_KEY")
+        self.host = os.environ.get("LANGFUSE_HOST", "https://cloud.langfuse.com")
+        self.client: Optional[Any] = None
+        self._init_client()
+
+    def _init_client(self) -> None:
+        if self.public_key and self.secret_key:
+            try:
+                self.client = Langfuse(
+                    public_key=self.public_key,
+                    secret_key=self.secret_key,
+                    host=self.host,
+                )
+            except Exception as e:
+                logger.warning("langfuse_client_init_failed", extra={"error": str(e)})
+                self.client = None
+
+    def record_trace(
+        self,
+        trace_id: str,
+        fingerprint: str,
+        source_url: str,
+        status: str,
+        spans_data: Dict[str, Dict[str, Any]],
+        generations: List[Dict[str, Any]],
+        metrics: Dict[str, Any],
+    ) -> bool:
+        """Records a trace with 8 stable spans, generations, and metrics.
+
+        If Langfuse is unavailable or unconfigured, gracefully queues to SQLite pending_telemetry.
+        """
+        payload = {
+            "trace_id": trace_id,
+            "fingerprint": fingerprint,
+            "source_url": source_url,
+            "status": status,
+            "spans": spans_data,
+            "generations": generations,
+            "metrics": metrics,
+            "timestamp": time.time(),
+        }
+
+        if not self.client:
+            event_id = str(uuid.uuid4())
+            self.store.queue_telemetry(
+                event_id=event_id,
+                fingerprint=fingerprint,
+                event_type="trace",
+                payload_json=json.dumps(payload),
+            )
+            return False
+
+        try:
+            # Emit directly to Langfuse
+            trace = self.client.trace(
+                id=trace_id,
+                name="article_consolidation",
+                metadata={
+                    "fingerprint": fingerprint,
+                    "source_url": source_url,
+                    "status": status,
+                    "config_version": self.config.config_version,
+                },
+                tags=[self.config.langfuse.environment, status],
+            )
+
+            for span_name, s_info in spans_data.items():
+                trace.span(
+                    name=span_name,
+                    start_time=s_info.get("start_time"),
+                    end_time=s_info.get("end_time"),
+                    metadata=s_info.get("metadata", {}),
+                    status_message=s_info.get("status", "SUCCESS"),
+                )
+
+            for gen in generations:
+                trace.generation(
+                    name=gen.get("name", "llm_call"),
+                    model=gen.get("model"),
+                    model_parameters=gen.get("model_parameters", {}),
+                    input=gen.get("input"),
+                    output=gen.get("output"),
+                    usage=gen.get("usage", {}),
+                    start_time=gen.get("start_time"),
+                    end_time=gen.get("end_time"),
+                )
+
+            self.client.flush()
+            return True
+        except Exception as e:
+            logger.warning("langfuse_emission_failed_queued_to_sqlite", extra={"error": str(e)})
+            event_id = str(uuid.uuid4())
+            self.store.queue_telemetry(
+                event_id=event_id,
+                fingerprint=fingerprint,
+                event_type="trace",
+                payload_json=json.dumps(payload),
+            )
+            return False

src/runtime/observability/structured_logger.py

@@ -0,0 +1,119 @@
+"""Structured JSON logging with secret sanitization and strict omission of sensitive/full payloads."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from typing import Any, Dict, Optional, Set
+
+KNOWN_SECRET_ENV_VARS = [
+    "GROQ_API_KEY",
+    "DEEPSEEK_API_KEY",
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "LANGFUSE_PUBLIC_KEY",
+    "LANGFUSE_SECRET_KEY",
+    "LANGFUSE_HOST",
+]
+
+
+class SanitizedJsonLogger:
+    def __init__(self, service_name: str = "article-consolidation-runtime"):
+        self.service_name = service_name
+        self.secret_values: Set[str] = set()
+        self._refresh_secrets()
+
+    def _refresh_secrets(self) -> None:
+        self.secret_values.clear()
+        for var in KNOWN_SECRET_ENV_VARS:
+            val = os.environ.get(var)
+            if val and len(val.strip()) > 3:
+                self.secret_values.add(val.strip())
+
+    def sanitize_text(self, text: str) -> str:
+        if not text:
+            return text
+        sanitized = text
+        for secret in self.secret_values:
+            if secret in sanitized:
+                sanitized = sanitized.replace(secret, "[REDACTED_SECRET]")
+        return sanitized
+
+    def sanitize_dict(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        result: Dict[str, Any] = {}
+        for k, v in data.items():
+            lower_k = str(k).lower()
+            if any(
+                s in lower_k
+                for s in ["authorization", "auth", "token", "secret", "api_key", "apikey"]
+            ):
+                result[k] = "[REDACTED_SECRET]"
+            elif isinstance(v, str):
+                result[k] = self.sanitize_text(v)
+            elif isinstance(v, dict):
+                result[k] = self.sanitize_dict(v)
+            elif isinstance(v, list):
+                result[k] = [
+                    self.sanitize_dict(item)
+                    if isinstance(item, dict)
+                    else self.sanitize_text(item)
+                    if isinstance(item, str)
+                    else item
+                    for item in v
+                ]
+            else:
+                result[k] = v
+        return result
+
+    def log(
+        self,
+        level: str,
+        event: str,
+        fingerprint: Optional[str] = None,
+        run_id: Optional[str] = None,
+        error_code: Optional[str] = None,
+        extra: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Emits a structured JSON log line to stderr."""
+        record: Dict[str, Any] = {
+            "timestamp": time.time(),
+            "service": self.service_name,
+            "level": level.upper(),
+            "event": event,
+        }
+        if fingerprint:
+            record["fingerprint"] = fingerprint
+        if run_id:
+            record["run_id"] = run_id
+        if error_code:
+            record["error_code"] = error_code
+        if extra:
+            # Ensure full ECP, full HTML, or full article text are not passed
+            safe_extra = self.sanitize_dict(extra)
+            safe_extra.pop("full_ecp", None)
+            safe_extra.pop("raw_html", None)
+            safe_extra.pop("full_body_text", None)
+            record["data"] = safe_extra
+
+        # Format as compact single-line JSON and write to stderr
+        try:
+            line = json.dumps(record, ensure_ascii=False)
+            sys.stderr.write(line + "\n")
+            sys.stderr.flush()
+        except Exception:
+            pass
+
+    def info(self, event: str, **kwargs: Any) -> None:
+        self.log("INFO", event, **kwargs)
+
+    def warning(self, event: str, **kwargs: Any) -> None:
+        self.log("WARN", event, **kwargs)
+
+    def error(self, event: str, **kwargs: Any) -> None:
+        self.log("ERROR", event, **kwargs)
+
+
+# Global default logger instance
+logger = SanitizedJsonLogger()

src/runtime/quality/invariants.py

@@ -0,0 +1,34 @@
+"""Release-wide critical quality assertion suite verifying the 11 critical invariants."""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+
+def verify_all_11_invariants(
+    metrics_summary: Dict[str, Any],
+) -> Dict[str, bool]:
+    """Evaluates the 11 release-wide critical invariants."""
+    results = {
+        "inv_1_zero_ungrounded_content": metrics_summary.get("ungrounded_content_count", 0) == 0,
+        "inv_2_zero_regex": metrics_summary.get("regex_violation_count", 0) == 0,
+        "inv_3_zero_powerful_models": metrics_summary.get("powerful_model_violation_count", 0) == 0,
+        "inv_4_zero_orphan_temp_files": metrics_summary.get("orphan_temp_files_count", 0) == 0,
+        "inv_5_markdown_hash_integrity_100pct": metrics_summary.get("hash_mismatches_count", 0)
+        == 0,
+        "inv_6_zero_markdown_on_ecp_rejection": metrics_summary.get(
+            "markdown_on_ecp_rejection_count", 0
+        )
+        == 0,
+        "inv_7_repairs_within_5_closed_categories": metrics_summary.get(
+            "unapproved_repairs_count", 0
+        )
+        == 0,
+        "inv_8_tags_normalized_and_bounded": metrics_summary.get("invalid_tags_count", 0) == 0,
+        "inv_9_manifest_schema_compliance_100pct": metrics_summary.get("invalid_manifests_count", 0)
+        == 0,
+        "inv_10_idempotency_compliance": metrics_summary.get("idempotency_failures_count", 0) == 0,
+        "inv_11_cost_budget_median_within_limit": metrics_summary.get("median_cost_usd", 0.0)
+        <= 0.0006,
+    }
+    return results

src/runtime/storage/file_store.py

@@ -0,0 +1,145 @@
+"""Atomic filesystem writer and shared manifest generator.
+
+Enforces:
+- Writing temporary files in the same destination directory/filesystem
+- Flushing, closing, and verifying SHA-256 content hash before renaming
+- Atomic rename via `os.replace` without cross-filesystem moves
+- Manifest generation adhering to manifest-output.schema.json and the 16 normative error codes
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+# Normative 16 Error / Rejection Codes
+APPROVED_ERROR_CODES = {
+    "INVALID_ARTICLE_SCHEMA",
+    "INVALID_ECP_SCHEMA",
+    "MISSING_SELECTED_EXTRACTOR",
+    "INVALID_SELECTED_EXTRACTOR",
+    "SELECTED_EXTRACTOR_UNAVAILABLE",
+    "MISSING_SOURCE_URL",
+    "MISSING_TITLE_CANDIDATE",
+    "MISSING_CONTENT",
+    "HYGIENE_FAILED",
+    "GROUNDING_VIOLATION",
+    "INVALID_TEXT_REPAIR",
+    "ECP_CLASSIFICATION_FAILED",
+    "ECP_REJECTED",
+    "ENRICHMENT_FAILED",
+    "PERSISTENCE_FAILED",
+    "TELEMETRY_PENDING",
+}
+
+
+def write_file_atomically(
+    target_path: Path | str, content: str | bytes, expected_hash: Optional[str] = None
+) -> Tuple[str, int]:
+    """Writes content atomically to target_path using a temporary file in the same directory.
+
+    Returns:
+        (sha256_hex, byte_count)
+    """
+    dest = Path(target_path).resolve()
+    dest.parent.mkdir(parents=True, exist_ok=True)
+
+    data_bytes = content.encode("utf-8") if isinstance(content, str) else content
+    actual_hash = hashlib.sha256(data_bytes).hexdigest()
+
+    if expected_hash and actual_hash != expected_hash:
+        raise ValueError(
+            f"Content hash mismatch! Expected {expected_hash}, calculated {actual_hash}"
+        )
+
+    # Create temporary file in the same directory to guarantee same filesystem for atomic rename
+    temp_fd, temp_path_str = tempfile.mkstemp(
+        dir=str(dest.parent), prefix=f".tmp_{dest.stem}_", suffix=".tmp"
+    )
+    temp_path = Path(temp_path_str)
+
+    try:
+        with os.fdopen(temp_fd, "wb") as f:
+            f.write(data_bytes)
+            f.flush()
+            os.fsync(f.fileno())
+
+        # Verify hash of written file
+        written_hash = hashlib.sha256(temp_path.read_bytes()).hexdigest()
+        if written_hash != actual_hash:
+            raise IOError(
+                f"Disk write verification failed: hash mismatch ({written_hash} != {actual_hash})"
+            )
+
+        # Atomic replacement
+        os.replace(str(temp_path), str(dest))
+    except Exception as e:
+        if temp_path.exists():
+            try:
+                temp_path.unlink()
+            except Exception:
+                pass
+        raise IOError(f"Atomic file write failed for {dest}: {e}") from e
+
+    return actual_hash, len(data_bytes)
+
+
+def create_manifest_dict(
+    fingerprint: str,
+    source_url: str,
+    selected_extractor: Optional[str],
+    final_status: str,  # completed_text, rejected_ecp, failed_validation, failed_processing
+    generate_markdown: bool,
+    config_version: str,
+    markdown_path: Optional[str] = None,
+    markdown_hash: Optional[str] = None,
+    ecp_classification: Optional[Dict[str, Any]] = None,
+    enrichment: Optional[Dict[str, Any]] = None,
+    provider_versions: Optional[Dict[str, Any]] = None,
+    model_versions: Optional[Dict[str, Any]] = None,
+    prompt_versions: Optional[Dict[str, Any]] = None,
+    trace_id: Optional[str] = None,
+    error_codes: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """Generates an output manifest adhering strictly to manifest-output.schema.json."""
+    codes = error_codes or []
+    for c in codes:
+        if c not in APPROVED_ERROR_CODES:
+            raise ValueError(f"Unapproved error code '{c}' in manifest generation.")
+
+    manifest: Dict[str, Any] = {
+        "schema_version": "1.0.0",
+        "fingerprint": fingerprint,
+        "source_url": source_url,
+        "selected_extractor": selected_extractor,
+        "final_status": final_status,
+        "generate_markdown": generate_markdown,
+        "markdown_path": markdown_path,
+        "markdown_hash": markdown_hash,
+        "ecp_classification": ecp_classification,
+        "enrichment": enrichment,
+        "provider_versions": provider_versions,
+        "model_versions": model_versions,
+        "prompt_versions": prompt_versions,
+        "config_version": config_version,
+        "trace_id": trace_id,
+        "error_codes": codes,
+    }
+    return manifest
+
+
+def persist_manifest_atomically(
+    output_dir: Path | str, manifest_dict: Dict[str, Any]
+) -> Tuple[Path, str]:
+    """Persists a manifest dictionary as <fingerprint>.result.json atomically."""
+    out_dir = Path(output_dir)
+    fingerprint = manifest_dict["fingerprint"]
+    manifest_path = out_dir / f"{fingerprint}.result.json"
+
+    manifest_json = json.dumps(manifest_dict, indent=2, ensure_ascii=False)
+    m_hash, _ = write_file_atomically(manifest_path, manifest_json)
+    return manifest_path, m_hash

src/runtime/storage/markdown_renderer.py

@@ -0,0 +1,65 @@
+"""Canonical YAML front matter and Markdown body renderer."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+from src.runtime.candidate.models import CandidateObject
+
+
+def render_canonical_markdown(
+    title: str,
+    subtitle: Optional[str],
+    fingerprint: str,
+    source_url: str,
+    published_date: Optional[str],
+    language: str,
+    sentiment: str,
+    tags: List[str],
+    ecp_target_id: str,
+    ecp_target_name: str,
+    body_blocks: List[CandidateObject],
+) -> str:
+    """Renders the final canonical Markdown document with YAML front matter."""
+    # Build YAML front matter dictionary
+    front_matter_dict: Dict[str, Any] = {
+        "title": title.strip(),
+        "fingerprint": fingerprint,
+        "source_url": source_url,
+        "language": language,
+        "sentiment": sentiment,
+        "tags": list(tags),
+        "ecp_target_id": ecp_target_id,
+        "ecp_target_name": ecp_target_name,
+    }
+    if published_date:
+        front_matter_dict["published_date"] = published_date
+
+    yaml_str = yaml.safe_dump(front_matter_dict, sort_keys=False, allow_unicode=True).strip()
+
+    # Build Markdown Body
+    body_parts: List[str] = [f"# {title.strip()}"]
+
+    if subtitle and subtitle.strip():
+        body_parts.append(f"*{subtitle.strip()}*")
+
+    for blk in body_blocks:
+        t = blk.text.strip()
+        if not t:
+            continue
+
+        if blk.type == "heading":
+            level = blk.level or 2
+            prefix = "#" * max(2, min(6, level))
+            body_parts.append(f"{prefix} {t}")
+        elif blk.type == "list_item":
+            body_parts.append(f"- {t}")
+        elif blk.type == "quote":
+            body_parts.append(f"> {t}")
+        else:
+            body_parts.append(t)
+
+    full_body = "\n\n".join(body_parts)
+    return f"---\n{yaml_str}\n---\n\n{full_body}\n"

src/runtime/storage/sqlite_store.py

@@ -0,0 +1,228 @@
+"""SQLite WAL storage, state persistence, atomic fingerprint claim, and native backup/restore."""
+
+from __future__ import annotations
+
+import sqlite3
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+
+class SQLiteStore:
+    def __init__(self, db_path: Path | str, busy_timeout_ms: int = 5000):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.busy_timeout_ms = busy_timeout_ms
+        self._init_db()
+
+    def _get_connection(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(self.db_path), timeout=self.busy_timeout_ms / 1000.0)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL;")
+        conn.execute(f"PRAGMA busy_timeout={self.busy_timeout_ms};")
+        conn.execute("PRAGMA foreign_keys=ON;")
+        return conn
+
+    def _init_db(self) -> None:
+        with self._get_connection() as conn:
+            conn.executescript("""
+                CREATE TABLE IF NOT EXISTS executions (
+                    fingerprint TEXT PRIMARY KEY,
+                    source_url TEXT NOT NULL,
+                    selected_extractor TEXT,
+                    current_status TEXT NOT NULL,
+                    final_status TEXT,
+                    error_codes TEXT, -- JSON array of error codes
+                    generate_markdown INTEGER NOT NULL DEFAULT 0,
+                    markdown_path TEXT,
+                    markdown_hash TEXT,
+                    manifest_path TEXT,
+                    manifest_hash TEXT,
+                    config_version TEXT NOT NULL,
+                    trace_id TEXT,
+                    created_at REAL NOT NULL,
+                    updated_at REAL NOT NULL,
+                    completed_at REAL
+                );
+
+                CREATE TABLE IF NOT EXISTS state_transitions (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    fingerprint TEXT NOT NULL,
+                    from_status TEXT NOT NULL,
+                    to_status TEXT NOT NULL,
+                    transition_time REAL NOT NULL,
+                    reason TEXT,
+                    FOREIGN KEY (fingerprint) REFERENCES executions(fingerprint)
+                );
+
+                CREATE TABLE IF NOT EXISTS pending_telemetry (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    event_id TEXT UNIQUE NOT NULL,
+                    fingerprint TEXT NOT NULL,
+                    event_type TEXT NOT NULL,
+                    payload_json TEXT NOT NULL,
+                    created_at REAL NOT NULL,
+                    flushed INTEGER NOT NULL DEFAULT 0,
+                    flushed_at REAL
+                );
+
+                CREATE INDEX IF NOT EXISTS idx_executions_status ON executions(current_status);
+                CREATE INDEX IF NOT EXISTS idx_pending_telemetry_flushed ON pending_telemetry(flushed);
+            """)
+
+    def claim_or_get_execution(
+        self,
+        fingerprint: str,
+        source_url: str,
+        selected_extractor: Optional[str],
+        config_version: str,
+    ) -> Tuple[bool, Dict[str, Any]]:
+        """Atomically claims execution for a fingerprint.
+
+        Returns:
+            (is_new_claim, execution_record)
+        """
+        now = time.time()
+        conn = self._get_connection()
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            cur = conn.cursor()
+            cur.execute("SELECT * FROM executions WHERE fingerprint = ?", (fingerprint,))
+            row = cur.fetchone()
+            if row:
+                conn.commit()
+                return False, dict(row)
+
+            cur.execute(
+                """
+                INSERT INTO executions (
+                    fingerprint, source_url, selected_extractor, current_status,
+                    config_version, created_at, updated_at
+                ) VALUES (?, ?, ?, 'received', ?, ?, ?)
+            """,
+                (fingerprint, source_url, selected_extractor, config_version, now, now),
+            )
+
+            cur.execute(
+                """
+                INSERT INTO state_transitions (fingerprint, from_status, to_status, transition_time, reason)
+                VALUES (?, 'none', 'received', ?, 'initial ingestion claim')
+            """,
+                (fingerprint, now),
+            )
+
+            cur.execute("SELECT * FROM executions WHERE fingerprint = ?", (fingerprint,))
+            res = dict(cur.fetchone())
+            conn.commit()
+            return True, res
+        except Exception:
+            try:
+                conn.rollback()
+            except Exception:
+                pass
+            with self._get_connection() as c2:
+                cur2 = c2.cursor()
+                cur2.execute("SELECT * FROM executions WHERE fingerprint = ?", (fingerprint,))
+                row2 = cur2.fetchone()
+                if row2:
+                    return False, dict(row2)
+            raise
+        finally:
+            conn.close()
+
+    def record_transition(
+        self,
+        fingerprint: str,
+        to_status: str,
+        reason: Optional[str] = None,
+        extra_fields: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Records a state transition for the given fingerprint atomically."""
+        now = time.time()
+        with self._get_connection() as conn:
+            cur = conn.cursor()
+            cur.execute(
+                "SELECT current_status FROM executions WHERE fingerprint = ?", (fingerprint,)
+            )
+            row = cur.fetchone()
+            if not row:
+                raise ValueError(f"Execution with fingerprint '{fingerprint}' does not exist.")
+
+            from_status = row["current_status"]
+
+            # Update execution record
+            set_clauses = ["current_status = ?", "updated_at = ?"]
+            params: List[Any] = [to_status, now]
+
+            if extra_fields:
+                for k, v in extra_fields.items():
+                    set_clauses.append(f"{k} = ?")
+                    params.append(v)
+
+            params.append(fingerprint)
+            cur.execute(
+                f"UPDATE executions SET {', '.join(set_clauses)} WHERE fingerprint = ?", params
+            )
+
+            # Log transition
+            cur.execute(
+                """
+                INSERT INTO state_transitions (fingerprint, from_status, to_status, transition_time, reason)
+                VALUES (?, ?, ?, ?, ?)
+            """,
+                (fingerprint, from_status, to_status, now, reason or ""),
+            )
+
+    def get_execution(self, fingerprint: str) -> Optional[Dict[str, Any]]:
+        with self._get_connection() as conn:
+            cur = conn.cursor()
+            cur.execute("SELECT * FROM executions WHERE fingerprint = ?", (fingerprint,))
+            row = cur.fetchone()
+            return dict(row) if row else None
+
+    def queue_telemetry(
+        self, event_id: str, fingerprint: str, event_type: str, payload_json: str
+    ) -> None:
+        now = time.time()
+        with self._get_connection() as conn:
+            conn.execute(
+                """
+                INSERT OR IGNORE INTO pending_telemetry (event_id, fingerprint, event_type, payload_json, created_at, flushed)
+                VALUES (?, ?, ?, ?, ?, 0)
+            """,
+                (event_id, fingerprint, event_type, payload_json, now),
+            )
+
+    def get_unflushed_telemetry(self, limit: int = 100) -> List[Dict[str, Any]]:
+        with self._get_connection() as conn:
+            cur = conn.cursor()
+            cur.execute(
+                "SELECT * FROM pending_telemetry WHERE flushed = 0 ORDER BY created_at ASC LIMIT ?",
+                (limit,),
+            )
+            return [dict(row) for row in cur.fetchall()]
+
+    def mark_telemetry_flushed(self, event_ids: List[str]) -> None:
+        now = time.time()
+        with self._get_connection() as conn:
+            conn.executemany(
+                "UPDATE pending_telemetry SET flushed = 1, flushed_at = ? WHERE event_id = ?",
+                [(now, eid) for eid in event_ids],
+            )
+
+    def backup_db(self, backup_path: Path | str) -> None:
+        """Native SQLite backup API creating a consistent point-in-time database snapshot."""
+        target_path = Path(backup_path)
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+        with self._get_connection() as src_conn:
+            with sqlite3.connect(str(target_path)) as dst_conn:
+                src_conn.backup(dst_conn)
+
+    def restore_db(self, source_backup_path: Path | str) -> None:
+        """Native SQLite restore API overwriting the current database from a snapshot."""
+        src_path = Path(source_backup_path)
+        if not src_path.exists():
+            raise FileNotFoundError(f"Backup file not found: {src_path}")
+        with sqlite3.connect(str(src_path)) as src_conn:
+            with self._get_connection() as dst_conn:
+                src_conn.backup(dst_conn)

src/tools/__init__.py

@@ -0,0 +1 @@
+"""Multilingual NLP tools and extractor utilities namespace."""

src/tools/adapters/base.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 
-from src.models import ClassificationResult, ECPSnapshot
+from src.tools.models import ClassificationResult, ECPSnapshot
 
 
 class BaseNLPAdapter(ABC):

src/tools/adapters/ecp/schemas/ecp-profile.schema.json

@@ -0,0 +1,52 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://schemas.aftech.internal/ecp/v1/ecp-profile.schema.json",
+  "title": "ECPProfileSchema",
+  "type": "object",
+  "required": [
+    "target_entity_id",
+    "target_name",
+    "aliases",
+    "domain",
+    "anchors"
+  ],
+  "properties": {
+    "target_entity_id": { "type": "string", "minLength": 1 },
+    "target_name": { "type": "string", "minLength": 1 },
+    "aliases": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "domain": { "type": "string", "minLength": 1 },
+    "anchors": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "negative_anchors": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "graph_version": { "type": "string" },
+    "related_entities": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["entity_id", "name", "relation_type", "weight"],
+        "properties": {
+          "entity_id": { "type": "string" },
+          "name": { "type": "string" },
+          "relation_type": { "type": "string" },
+          "weight": { "type": "number" },
+          "aliases": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "scope": { "type": "string" },
+          "confidence": { "type": "number" }
+        },
+        "additionalProperties": false
+      }
+    }
+  },
+  "additionalProperties": false
+}

src/tools/adapters/embeddings.py

@@ -6,8 +6,8 @@ without requiring sentence-transformers to be pre-installed in the core environm
 
 from __future__ import annotations
 
-from src.adapters.base import BaseNLPAdapter
-from src.models import ClassificationResult, ECPSnapshot
+from src.tools.adapters.base import BaseNLPAdapter
+from src.tools.models import ClassificationResult, ECPSnapshot
 
 
 class LocalEmbeddingsAdapter(BaseNLPAdapter):

src/tools/adapters/llm.py

@@ -13,8 +13,8 @@ import urllib.request
 from pathlib import Path
 from typing import Callable, Optional
 
-from src.adapters.base import BaseNLPAdapter
-from src.models import ClassificationResult, DecisionCategory, ECPSnapshot
+from src.tools.adapters.base import BaseNLPAdapter
+from src.tools.models import ClassificationResult, DecisionCategory, ECPSnapshot
 
 
 def _load_env_file() -> None:

src/tools/classifier.py

@@ -5,13 +5,13 @@ from __future__ import annotations
 import re
 from typing import Any, Optional
 
-from src.language import detect_language, normalize_text
-from src.models import (
+from src.tools.language import detect_language, normalize_text
+from src.tools.models import (
     ClassificationResult,
     DecisionCategory,
     ECPSnapshot,
 )
-from src.parser import extract_evidence_snippets, strip_markdown
+from src.tools.parser import extract_evidence_snippets, strip_markdown
 
 
 def match_phrase_in_text(phrase: str, normalized_text: str) -> bool:
@@ -53,12 +53,12 @@ class InherenceClassifier:
         self._llm_adapter = llm_adapter
 
         if enable_embeddings:
-            from src.adapters.embeddings import LocalEmbeddingsAdapter
+            from src.tools.adapters.embeddings import LocalEmbeddingsAdapter
 
             self._embeddings_adapter = LocalEmbeddingsAdapter()
 
         if self.enable_llm and self._llm_adapter is None:
-            from src.adapters.llm import LLMFallbackAdapter
+            from src.tools.adapters.llm import LLMFallbackAdapter
 
             self._llm_adapter = LLMFallbackAdapter()