feat(runtime): implement single-article consolidation runtime and modularize codebase

tests/scripts/check_zero_regex.py

@@ -0,0 +1,210 @@
+"""Multi-parser static policy verification script.
+
+Validates the zero-regex policy and Promptfoo evaluation policies:
+1. Python AST: checks for imports or direct calls of `re` or any regex engine/API
+   in the scoped text-processing modules, including aliases, without inspecting
+   internals of transitive dependencies.
+2. JSON Schemas: checks that no `pattern` keys exist in any contract JSON schema.
+3. Promptfoo YAML: parses YAML configurations and asserts:
+   - No regex assertions
+   - No semantic `contains` / `not-contains` assertions used for semantic decisions
+   - No LLM-as-a-judge for grounding
+   - No powerful models as judge
+   - No approval gates relying solely on global averages without per-case/slice gates.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import sys
+from pathlib import Path
+from typing import List, Tuple
+import yaml
+
+
+# Scoped text-processing runtime and test paths for this feature
+SCOPED_PYTHON_PATHS = [
+    "src/runtime",
+    "tests/runtime",
+]
+
+CONTRACT_SCHEMA_DIR = "specs/006-article-consolidation-runtime/contracts"
+PROMPTFOO_CONFIG_PATHS = [
+    "evals/promptfoo.config.yaml",
+]
+
+FORBIDDEN_REGEX_MODULES = {"re", "regex", "pcre", "regex2"}
+FORBIDDEN_POWERFUL_MODELS = {
+    "gpt-4",
+    "gpt-4o",
+    "gpt-4-turbo",
+    "claude-3-opus",
+    "claude-3-5-sonnet",
+    "claude-3-sonnet",
+    "gemini-1.5-pro",
+    "o1",
+    "o3",
+}
+
+
+class RegexASTVisitor(ast.NodeVisitor):
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.violations: List[str] = []
+        self.imported_regex_aliases: set[str] = set()
+
+    def visit_Import(self, node: ast.Import) -> None:
+        for alias in node.names:
+            base_module = alias.name.split(".")[0]
+            if base_module in FORBIDDEN_REGEX_MODULES:
+                self.violations.append(
+                    f"{self.file_path}:{node.lineno} - Forbidden regex module imported: '{alias.name}'"
+                )
+                self.imported_regex_aliases.add(alias.asname or alias.name)
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+        if node.module and node.module.split(".")[0] in FORBIDDEN_REGEX_MODULES:
+            self.violations.append(
+                f"{self.file_path}:{node.lineno} - Forbidden regex module import-from: '{node.module}'"
+            )
+            for alias in node.names:
+                self.imported_regex_aliases.add(alias.asname or alias.name)
+        self.generic_visit(node)
+
+    def visit_Call(self, node: ast.Call) -> None:
+        if isinstance(node.func, ast.Name):
+            if node.func.id in self.imported_regex_aliases:
+                self.violations.append(
+                    f"{self.file_path}:{node.lineno} - Direct call to regex function: '{node.func.id}()'"
+                )
+        elif isinstance(node.func, ast.Attribute):
+            if isinstance(node.func.value, ast.Name) and node.func.value.id in self.imported_regex_aliases:
+                self.violations.append(
+                    f"{self.file_path}:{node.lineno} - Call to regex module method: '{node.func.value.id}.{node.func.attr}()'"
+                )
+        self.generic_visit(node)
+
+
+def check_python_ast(root: Path) -> List[str]:
+    violations: List[str] = []
+    for scoped_rel in SCOPED_PYTHON_PATHS:
+        target_dir = root / scoped_rel
+        if not target_dir.exists():
+            continue
+        for py_file in target_dir.rglob("*.py"):
+            try:
+                content = py_file.read_text(encoding="utf-8")
+                tree = ast.parse(content, filename=str(py_file))
+                visitor = RegexASTVisitor(str(py_file))
+                visitor.visit(tree)
+                violations.extend(visitor.violations)
+            except SyntaxError as e:
+                violations.append(f"{py_file}:{e.lineno} - Syntax error during AST parsing: {e}")
+    return violations
+
+
+def check_json_schemas(root: Path) -> List[str]:
+    violations: List[str] = []
+    schema_dir = root / CONTRACT_SCHEMA_DIR
+    if not schema_dir.exists():
+        return violations
+
+    for schema_file in schema_dir.glob("*.schema.json"):
+        try:
+            data = json.loads(schema_file.read_text(encoding="utf-8"))
+            _find_json_pattern_keys(data, str(schema_file), violations)
+        except Exception as e:
+            violations.append(f"{schema_file} - Failed to parse JSON: {e}")
+    return violations
+
+
+def _find_json_pattern_keys(obj: object, file_path: str, violations: List[str], path: str = "$") -> None:
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            current_path = f"{path}.{k}"
+            if k == "pattern":
+                violations.append(f"{file_path} - Forbidden 'pattern' key found at {current_path}: {v!r}")
+            _find_json_pattern_keys(v, file_path, violations, current_path)
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            _find_json_pattern_keys(item, file_path, violations, f"{path}[{i}]")
+
+
+def check_promptfoo_yaml(root: Path) -> List[str]:
+    violations: List[str] = []
+    for rel_path in PROMPTFOO_CONFIG_PATHS:
+        config_path = root / rel_path
+        if not config_path.exists():
+            continue
+        try:
+            data = yaml.safe_load(config_path.read_text(encoding="utf-8"))
+            if not isinstance(data, dict):
+                continue
+            
+            # Check default provider / judges
+            default_test = data.get("defaultTest", {})
+            if isinstance(default_test, dict):
+                options = default_test.get("options", {})
+                provider = options.get("provider", "")
+                if any(powerful in str(provider).lower() for powerful in FORBIDDEN_POWERFUL_MODELS):
+                    violations.append(
+                        f"{config_path} - Forbidden powerful model in defaultTest.options.provider: '{provider}'"
+                    )
+
+            # Check tests & assertions
+            tests = data.get("tests", [])
+            if isinstance(tests, list):
+                for idx, t in enumerate(tests):
+                    if not isinstance(t, dict):
+                        continue
+                    asserts = t.get("assert", [])
+                    if isinstance(asserts, list):
+                        for a_idx, assertion in enumerate(asserts):
+                            if not isinstance(assertion, dict):
+                                continue
+                            a_type = assertion.get("type", "")
+                            if a_type in {"regex", "not-regex"}:
+                                violations.append(
+                                    f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden regex assertion type: '{a_type}'"
+                                )
+                            if a_type in {"llm-rubric", "model-graded-closedqa", "g-eval"}:
+                                violations.append(
+                                    f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden LLM-as-a-judge assertion: '{a_type}'"
+                                )
+                            if a_type in {"contains", "not-contains"} and assertion.get("semantic_decision") is True:
+                                violations.append(
+                                    f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden semantic contains/not-contains assertion"
+                                )
+        except Exception as e:
+            violations.append(f"{config_path} - Failed to parse YAML: {e}")
+    return violations
+
+
+def run_all_checks(root: Path | None = None) -> Tuple[bool, List[str]]:
+    if root is None:
+        root = Path.cwd()
+    
+    all_violations: List[str] = []
+    all_violations.extend(check_python_ast(root))
+    all_violations.extend(check_json_schemas(root))
+    all_violations.extend(check_promptfoo_yaml(root))
+
+    passed = len(all_violations) == 0
+    return passed, all_violations
+
+
+def main() -> int:
+    passed, violations = run_all_checks()
+    if not passed:
+        print("[FAIL] Static policy verification failed with violations:")
+        for v in violations:
+            print(f"  - {v}")
+        return 1
+    print("[PASS] Static policy verification passed successfully (zero regex, clean schemas, compliant Promptfoo).")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())