"""Multi-parser static policy verification script. Validates the zero-regex policy and Promptfoo evaluation policies: 1. Python AST: checks for imports or direct calls of `re` or any regex engine/API in the scoped text-processing modules, including aliases, without inspecting internals of transitive dependencies. 2. JSON Schemas: checks that no `pattern` keys exist in any contract JSON schema. 3. Promptfoo YAML: parses YAML configurations and asserts: - No regex assertions - No semantic `contains` / `not-contains` assertions used for semantic decisions - No LLM-as-a-judge for grounding - No powerful models as judge - No approval gates relying solely on global averages without per-case/slice gates. """ from __future__ import annotations import ast import json import sys from pathlib import Path from typing import List, Tuple import yaml # Scoped text-processing runtime and test paths for this feature SCOPED_PYTHON_PATHS = [ "src/runtime", "tests/runtime", ] CONTRACT_SCHEMA_DIR = "specs/006-article-consolidation-runtime/contracts" PROMPTFOO_CONFIG_PATHS = [ "evals/promptfoo.config.yaml", ] FORBIDDEN_REGEX_MODULES = {"re", "regex", "pcre", "regex2"} FORBIDDEN_POWERFUL_MODELS = { "gpt-4", "gpt-4o", "gpt-4-turbo", "claude-3-opus", "claude-3-5-sonnet", "claude-3-sonnet", "gemini-1.5-pro", "o1", "o3", } class RegexASTVisitor(ast.NodeVisitor): def __init__(self, file_path: str): self.file_path = file_path self.violations: List[str] = [] self.imported_regex_aliases: set[str] = set() def visit_Import(self, node: ast.Import) -> None: for alias in node.names: base_module = alias.name.split(".")[0] if base_module in FORBIDDEN_REGEX_MODULES: self.violations.append( f"{self.file_path}:{node.lineno} - Forbidden regex module imported: '{alias.name}'" ) self.imported_regex_aliases.add(alias.asname or alias.name) self.generic_visit(node) def visit_ImportFrom(self, node: ast.ImportFrom) -> None: if node.module and node.module.split(".")[0] in FORBIDDEN_REGEX_MODULES: self.violations.append( f"{self.file_path}:{node.lineno} - Forbidden regex module import-from: '{node.module}'" ) for alias in node.names: self.imported_regex_aliases.add(alias.asname or alias.name) self.generic_visit(node) def visit_Call(self, node: ast.Call) -> None: if isinstance(node.func, ast.Name): if node.func.id in self.imported_regex_aliases: self.violations.append( f"{self.file_path}:{node.lineno} - Direct call to regex function: '{node.func.id}()'" ) elif isinstance(node.func, ast.Attribute): if isinstance(node.func.value, ast.Name) and node.func.value.id in self.imported_regex_aliases: self.violations.append( f"{self.file_path}:{node.lineno} - Call to regex module method: '{node.func.value.id}.{node.func.attr}()'" ) self.generic_visit(node) def check_python_ast(root: Path) -> List[str]: violations: List[str] = [] for scoped_rel in SCOPED_PYTHON_PATHS: target_dir = root / scoped_rel if not target_dir.exists(): continue for py_file in target_dir.rglob("*.py"): try: content = py_file.read_text(encoding="utf-8") tree = ast.parse(content, filename=str(py_file)) visitor = RegexASTVisitor(str(py_file)) visitor.visit(tree) violations.extend(visitor.violations) except SyntaxError as e: violations.append(f"{py_file}:{e.lineno} - Syntax error during AST parsing: {e}") return violations def check_json_schemas(root: Path) -> List[str]: violations: List[str] = [] schema_dir = root / CONTRACT_SCHEMA_DIR if not schema_dir.exists(): return violations for schema_file in schema_dir.glob("*.schema.json"): try: data = json.loads(schema_file.read_text(encoding="utf-8")) _find_json_pattern_keys(data, str(schema_file), violations) except Exception as e: violations.append(f"{schema_file} - Failed to parse JSON: {e}") return violations def _find_json_pattern_keys(obj: object, file_path: str, violations: List[str], path: str = "$") -> None: if isinstance(obj, dict): for k, v in obj.items(): current_path = f"{path}.{k}" if k == "pattern": violations.append(f"{file_path} - Forbidden 'pattern' key found at {current_path}: {v!r}") _find_json_pattern_keys(v, file_path, violations, current_path) elif isinstance(obj, list): for i, item in enumerate(obj): _find_json_pattern_keys(item, file_path, violations, f"{path}[{i}]") def check_promptfoo_yaml(root: Path) -> List[str]: violations: List[str] = [] for rel_path in PROMPTFOO_CONFIG_PATHS: config_path = root / rel_path if not config_path.exists(): continue try: data = yaml.safe_load(config_path.read_text(encoding="utf-8")) if not isinstance(data, dict): continue # Check default provider / judges default_test = data.get("defaultTest", {}) if isinstance(default_test, dict): options = default_test.get("options", {}) provider = options.get("provider", "") if any(powerful in str(provider).lower() for powerful in FORBIDDEN_POWERFUL_MODELS): violations.append( f"{config_path} - Forbidden powerful model in defaultTest.options.provider: '{provider}'" ) # Check tests & assertions tests = data.get("tests", []) if isinstance(tests, list): for idx, t in enumerate(tests): if not isinstance(t, dict): continue asserts = t.get("assert", []) if isinstance(asserts, list): for a_idx, assertion in enumerate(asserts): if not isinstance(assertion, dict): continue a_type = assertion.get("type", "") if a_type in {"regex", "not-regex"}: violations.append( f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden regex assertion type: '{a_type}'" ) if a_type in {"llm-rubric", "model-graded-closedqa", "g-eval"}: violations.append( f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden LLM-as-a-judge assertion: '{a_type}'" ) if a_type in {"contains", "not-contains"} and assertion.get("semantic_decision") is True: violations.append( f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden semantic contains/not-contains assertion" ) except Exception as e: violations.append(f"{config_path} - Failed to parse YAML: {e}") return violations def run_all_checks(root: Path | None = None) -> Tuple[bool, List[str]]: if root is None: root = Path.cwd() all_violations: List[str] = [] all_violations.extend(check_python_ast(root)) all_violations.extend(check_json_schemas(root)) all_violations.extend(check_promptfoo_yaml(root)) passed = len(all_violations) == 0 return passed, all_violations def main() -> int: passed, violations = run_all_checks() if not passed: print("[FAIL] Static policy verification failed with violations:") for v in violations: print(f" - {v}") return 1 print("[PASS] Static policy verification passed successfully (zero regex, clean schemas, compliant Promptfoo).") return 0 if __name__ == "__main__": sys.exit(main())