feat(extractor): implement multi-engine article content extractor

- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability)
- Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing)
- Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts
- Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
2026-08-20 19:22:20 -03:00
parent 6e3d57619b
commit 6a45368cb0
85 changed files with 18345 additions and 3897 deletions
+9 -9
View File
@@ -1,15 +1,15 @@
"""Unit tests for ECP models, schema validation, and structured error handling."""
import pytest
from src.models import (
ECPSnapshot,
RelatedEntity,
ClassificationResult,
ClassificationError,
ClassificationResult,
DecisionCategory,
ECPSnapshot,
ErrorCode,
)
from src.parser import strip_markdown, extract_sentences, extract_evidence_snippets
from src.parser import extract_evidence_snippets, strip_markdown
def test_ecp_snapshot_valid():
@@ -29,9 +29,9 @@ def test_ecp_snapshot_valid():
"weight": 0.9,
"aliases": ["Transpetro Logística"],
"scope": "logistics",
"confidence": 0.95
"confidence": 0.95,
}
]
],
}
snapshot = ECPSnapshot.from_dict(data)
assert snapshot.target_entity_id == "ent_123"
@@ -48,7 +48,7 @@ def test_ecp_snapshot_defaults():
"target_name": "Petrobras",
"aliases": ["Petrobras"],
"domain": "Oil & Gas",
"anchors": ["energia"]
"anchors": ["energia"],
}
snapshot = ECPSnapshot.from_dict(data)
assert snapshot.negative_anchors == []
@@ -61,7 +61,7 @@ def test_ecp_snapshot_missing_required():
"target_entity_id": "ent_123",
"aliases": ["Petrobras"],
"domain": "Oil & Gas",
"anchors": ["energia"]
"anchors": ["energia"],
}
with pytest.raises(ValueError, match="Missing required field"):
ECPSnapshot.from_dict(data)
@@ -89,7 +89,7 @@ def test_classification_error_serialization():
err = ClassificationError(
error_code=ErrorCode.INVALID_ECP_JSON,
message="Malformed JSON syntax",
details={"path": "snapshot.json"}
details={"path": "snapshot.json"},
)
d = err.to_dict()
assert d["error_code"] == "invalid_ecp_json"