211 lines
8.2 KiB
Python
211 lines
8.2 KiB
Python
"""Multi-parser static policy verification script.
|
|
|
|
Validates the zero-regex policy and Promptfoo evaluation policies:
|
|
1. Python AST: checks for imports or direct calls of `re` or any regex engine/API
|
|
in the scoped text-processing modules, including aliases, without inspecting
|
|
internals of transitive dependencies.
|
|
2. JSON Schemas: checks that no `pattern` keys exist in any contract JSON schema.
|
|
3. Promptfoo YAML: parses YAML configurations and asserts:
|
|
- No regex assertions
|
|
- No semantic `contains` / `not-contains` assertions used for semantic decisions
|
|
- No LLM-as-a-judge for grounding
|
|
- No powerful models as judge
|
|
- No approval gates relying solely on global averages without per-case/slice gates.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import List, Tuple
|
|
import yaml
|
|
|
|
|
|
# Scoped text-processing runtime and test paths for this feature
|
|
SCOPED_PYTHON_PATHS = [
|
|
"src/runtime",
|
|
"tests/runtime",
|
|
]
|
|
|
|
CONTRACT_SCHEMA_DIR = "specs/006-article-consolidation-runtime/contracts"
|
|
PROMPTFOO_CONFIG_PATHS = [
|
|
"evals/promptfoo.config.yaml",
|
|
]
|
|
|
|
FORBIDDEN_REGEX_MODULES = {"re", "regex", "pcre", "regex2"}
|
|
FORBIDDEN_POWERFUL_MODELS = {
|
|
"gpt-4",
|
|
"gpt-4o",
|
|
"gpt-4-turbo",
|
|
"claude-3-opus",
|
|
"claude-3-5-sonnet",
|
|
"claude-3-sonnet",
|
|
"gemini-1.5-pro",
|
|
"o1",
|
|
"o3",
|
|
}
|
|
|
|
|
|
class RegexASTVisitor(ast.NodeVisitor):
|
|
def __init__(self, file_path: str):
|
|
self.file_path = file_path
|
|
self.violations: List[str] = []
|
|
self.imported_regex_aliases: set[str] = set()
|
|
|
|
def visit_Import(self, node: ast.Import) -> None:
|
|
for alias in node.names:
|
|
base_module = alias.name.split(".")[0]
|
|
if base_module in FORBIDDEN_REGEX_MODULES:
|
|
self.violations.append(
|
|
f"{self.file_path}:{node.lineno} - Forbidden regex module imported: '{alias.name}'"
|
|
)
|
|
self.imported_regex_aliases.add(alias.asname or alias.name)
|
|
self.generic_visit(node)
|
|
|
|
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
|
if node.module and node.module.split(".")[0] in FORBIDDEN_REGEX_MODULES:
|
|
self.violations.append(
|
|
f"{self.file_path}:{node.lineno} - Forbidden regex module import-from: '{node.module}'"
|
|
)
|
|
for alias in node.names:
|
|
self.imported_regex_aliases.add(alias.asname or alias.name)
|
|
self.generic_visit(node)
|
|
|
|
def visit_Call(self, node: ast.Call) -> None:
|
|
if isinstance(node.func, ast.Name):
|
|
if node.func.id in self.imported_regex_aliases:
|
|
self.violations.append(
|
|
f"{self.file_path}:{node.lineno} - Direct call to regex function: '{node.func.id}()'"
|
|
)
|
|
elif isinstance(node.func, ast.Attribute):
|
|
if isinstance(node.func.value, ast.Name) and node.func.value.id in self.imported_regex_aliases:
|
|
self.violations.append(
|
|
f"{self.file_path}:{node.lineno} - Call to regex module method: '{node.func.value.id}.{node.func.attr}()'"
|
|
)
|
|
self.generic_visit(node)
|
|
|
|
|
|
def check_python_ast(root: Path) -> List[str]:
|
|
violations: List[str] = []
|
|
for scoped_rel in SCOPED_PYTHON_PATHS:
|
|
target_dir = root / scoped_rel
|
|
if not target_dir.exists():
|
|
continue
|
|
for py_file in target_dir.rglob("*.py"):
|
|
try:
|
|
content = py_file.read_text(encoding="utf-8")
|
|
tree = ast.parse(content, filename=str(py_file))
|
|
visitor = RegexASTVisitor(str(py_file))
|
|
visitor.visit(tree)
|
|
violations.extend(visitor.violations)
|
|
except SyntaxError as e:
|
|
violations.append(f"{py_file}:{e.lineno} - Syntax error during AST parsing: {e}")
|
|
return violations
|
|
|
|
|
|
def check_json_schemas(root: Path) -> List[str]:
|
|
violations: List[str] = []
|
|
schema_dir = root / CONTRACT_SCHEMA_DIR
|
|
if not schema_dir.exists():
|
|
return violations
|
|
|
|
for schema_file in schema_dir.glob("*.schema.json"):
|
|
try:
|
|
data = json.loads(schema_file.read_text(encoding="utf-8"))
|
|
_find_json_pattern_keys(data, str(schema_file), violations)
|
|
except Exception as e:
|
|
violations.append(f"{schema_file} - Failed to parse JSON: {e}")
|
|
return violations
|
|
|
|
|
|
def _find_json_pattern_keys(obj: object, file_path: str, violations: List[str], path: str = "$") -> None:
|
|
if isinstance(obj, dict):
|
|
for k, v in obj.items():
|
|
current_path = f"{path}.{k}"
|
|
if k == "pattern":
|
|
violations.append(f"{file_path} - Forbidden 'pattern' key found at {current_path}: {v!r}")
|
|
_find_json_pattern_keys(v, file_path, violations, current_path)
|
|
elif isinstance(obj, list):
|
|
for i, item in enumerate(obj):
|
|
_find_json_pattern_keys(item, file_path, violations, f"{path}[{i}]")
|
|
|
|
|
|
def check_promptfoo_yaml(root: Path) -> List[str]:
|
|
violations: List[str] = []
|
|
for rel_path in PROMPTFOO_CONFIG_PATHS:
|
|
config_path = root / rel_path
|
|
if not config_path.exists():
|
|
continue
|
|
try:
|
|
data = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict):
|
|
continue
|
|
|
|
# Check default provider / judges
|
|
default_test = data.get("defaultTest", {})
|
|
if isinstance(default_test, dict):
|
|
options = default_test.get("options", {})
|
|
provider = options.get("provider", "")
|
|
if any(powerful in str(provider).lower() for powerful in FORBIDDEN_POWERFUL_MODELS):
|
|
violations.append(
|
|
f"{config_path} - Forbidden powerful model in defaultTest.options.provider: '{provider}'"
|
|
)
|
|
|
|
# Check tests & assertions
|
|
tests = data.get("tests", [])
|
|
if isinstance(tests, list):
|
|
for idx, t in enumerate(tests):
|
|
if not isinstance(t, dict):
|
|
continue
|
|
asserts = t.get("assert", [])
|
|
if isinstance(asserts, list):
|
|
for a_idx, assertion in enumerate(asserts):
|
|
if not isinstance(assertion, dict):
|
|
continue
|
|
a_type = assertion.get("type", "")
|
|
if a_type in {"regex", "not-regex"}:
|
|
violations.append(
|
|
f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden regex assertion type: '{a_type}'"
|
|
)
|
|
if a_type in {"llm-rubric", "model-graded-closedqa", "g-eval"}:
|
|
violations.append(
|
|
f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden LLM-as-a-judge assertion: '{a_type}'"
|
|
)
|
|
if a_type in {"contains", "not-contains"} and assertion.get("semantic_decision") is True:
|
|
violations.append(
|
|
f"{config_path}:tests[{idx}].assert[{a_idx}] - Forbidden semantic contains/not-contains assertion"
|
|
)
|
|
except Exception as e:
|
|
violations.append(f"{config_path} - Failed to parse YAML: {e}")
|
|
return violations
|
|
|
|
|
|
def run_all_checks(root: Path | None = None) -> Tuple[bool, List[str]]:
|
|
if root is None:
|
|
root = Path.cwd()
|
|
|
|
all_violations: List[str] = []
|
|
all_violations.extend(check_python_ast(root))
|
|
all_violations.extend(check_json_schemas(root))
|
|
all_violations.extend(check_promptfoo_yaml(root))
|
|
|
|
passed = len(all_violations) == 0
|
|
return passed, all_violations
|
|
|
|
|
|
def main() -> int:
|
|
passed, violations = run_all_checks()
|
|
if not passed:
|
|
print("[FAIL] Static policy verification failed with violations:")
|
|
for v in violations:
|
|
print(f" - {v}")
|
|
return 1
|
|
print("[PASS] Static policy verification passed successfully (zero regex, clean schemas, compliant Promptfoo).")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|