83 lines
2.8 KiB
JSON
83 lines
2.8 KiB
JSON
{
|
|
"test_execution_timestamp": "2026-08-24T02:34:30Z",
|
|
"methodology": "skill-suite-tests 4-axis risk-guided quality framework",
|
|
"summary": {
|
|
"total_tests": 319,
|
|
"passed": 319,
|
|
"failed": 0,
|
|
"skipped": 0,
|
|
"runtime_suite_tests": 72,
|
|
"tools_suite_tests": 247,
|
|
"live_real_api_e2e_tests": 2,
|
|
"static_checks": "PASS",
|
|
"zero_regex_compliance": "100%",
|
|
"cheap_models_enforcement": "100%"
|
|
},
|
|
"live_e2e_evidence": {
|
|
"endpoint": "https://omniroute.app.andreferraro.com/v1",
|
|
"model": "cgpt-web/gpt-5.5 / gpt-4o-mini",
|
|
"runtime_pipeline_execution": {
|
|
"article_input": "examples/sample_article_valid.json",
|
|
"ecp_snapshot": "examples/sample_ecp_snapshot.json",
|
|
"fingerprint": "c987f362b35e76239e3fda0841c9e5f89c3d4193760738e6a2e9424ce45fde88",
|
|
"final_status": "completed_text",
|
|
"markdown_sha256": "b6b21b19f10feab12525030016a3eeb4ed702cdec6d39c91fc42289b65091e0e",
|
|
"ecp_classification": "DIRECT_INHERENT (confidence: 0.98)",
|
|
"enrichment_sentiment": "positive",
|
|
"enrichment_tags": ["river plate", "copa sudamericana", "futebol"],
|
|
"execution_exit_code": 0
|
|
}
|
|
},
|
|
"coverage_axes": {
|
|
"purpose": [
|
|
"functional_correctness",
|
|
"regression_guard",
|
|
"contract_parity",
|
|
"security_isolation",
|
|
"performance_throughput",
|
|
"fault_resilience",
|
|
"live_real_api_e2e_verification"
|
|
],
|
|
"levels": [
|
|
"unit",
|
|
"contract",
|
|
"integration",
|
|
"fault_injection",
|
|
"quality",
|
|
"security",
|
|
"load",
|
|
"live_real_e2e_subprocess"
|
|
],
|
|
"quality_attributes": [
|
|
"reliability",
|
|
"determinism",
|
|
"concurrency_atomic_claims",
|
|
"zero_deadlocks",
|
|
"data_integrity",
|
|
"crash_recovery",
|
|
"budget_compliance",
|
|
"live_llm_inference_parity"
|
|
],
|
|
"profiles": [
|
|
"live_endpoint_omniroute_real_call",
|
|
"8_worker_high_contention_thread_barrier",
|
|
"corrupted_response_failover",
|
|
"rate_limit_429_exponential_backoff",
|
|
"server_error_500_fallback",
|
|
"offline_telemetry_degradation",
|
|
"prompt_injection_passive_treatment"
|
|
]
|
|
},
|
|
"invariants_verified": [
|
|
"INV-01: Zero regex across text processing modules",
|
|
"INV-02: Zero powerful models across runtime configurations",
|
|
"INV-03: Atomic file and manifest persistence (temp file + os.replace)",
|
|
"INV-04: SQLite WAL concurrency with BEGIN IMMEDIATE transactions",
|
|
"INV-05: Bidirectional crash recovery and manifest hash verification",
|
|
"INV-06: Strict CLI exit codes (0: success, 1: schema/arg error, 2: preflight error)",
|
|
"INV-07: Offline telemetry queue with sqlite storage and backoff",
|
|
"INV-08: Reference Golden Set 20/20 articles processing with zero errors",
|
|
"INV-09: Live E2E Real API Execution producing valid Markdown + Manifest on disk"
|
|
]
|
|
}
|