feat(extractor): implement multi-engine article content extractor
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
@@ -1,15 +1,15 @@
|
||||
"""Unit tests for ECP models, schema validation, and structured error handling."""
|
||||
|
||||
import pytest
|
||||
|
||||
from src.models import (
|
||||
ECPSnapshot,
|
||||
RelatedEntity,
|
||||
ClassificationResult,
|
||||
ClassificationError,
|
||||
ClassificationResult,
|
||||
DecisionCategory,
|
||||
ECPSnapshot,
|
||||
ErrorCode,
|
||||
)
|
||||
from src.parser import strip_markdown, extract_sentences, extract_evidence_snippets
|
||||
from src.parser import extract_evidence_snippets, strip_markdown
|
||||
|
||||
|
||||
def test_ecp_snapshot_valid():
|
||||
@@ -29,9 +29,9 @@ def test_ecp_snapshot_valid():
|
||||
"weight": 0.9,
|
||||
"aliases": ["Transpetro Logística"],
|
||||
"scope": "logistics",
|
||||
"confidence": 0.95
|
||||
"confidence": 0.95,
|
||||
}
|
||||
]
|
||||
],
|
||||
}
|
||||
snapshot = ECPSnapshot.from_dict(data)
|
||||
assert snapshot.target_entity_id == "ent_123"
|
||||
@@ -48,7 +48,7 @@ def test_ecp_snapshot_defaults():
|
||||
"target_name": "Petrobras",
|
||||
"aliases": ["Petrobras"],
|
||||
"domain": "Oil & Gas",
|
||||
"anchors": ["energia"]
|
||||
"anchors": ["energia"],
|
||||
}
|
||||
snapshot = ECPSnapshot.from_dict(data)
|
||||
assert snapshot.negative_anchors == []
|
||||
@@ -61,7 +61,7 @@ def test_ecp_snapshot_missing_required():
|
||||
"target_entity_id": "ent_123",
|
||||
"aliases": ["Petrobras"],
|
||||
"domain": "Oil & Gas",
|
||||
"anchors": ["energia"]
|
||||
"anchors": ["energia"],
|
||||
}
|
||||
with pytest.raises(ValueError, match="Missing required field"):
|
||||
ECPSnapshot.from_dict(data)
|
||||
@@ -89,7 +89,7 @@ def test_classification_error_serialization():
|
||||
err = ClassificationError(
|
||||
error_code=ErrorCode.INVALID_ECP_JSON,
|
||||
message="Malformed JSON syntax",
|
||||
details={"path": "snapshot.json"}
|
||||
details={"path": "snapshot.json"},
|
||||
)
|
||||
d = err.to_dict()
|
||||
assert d["error_code"] == "invalid_ecp_json"
|
||||
|
||||
Reference in New Issue
Block a user