feat(extractor): implement multi-engine article content extractor

- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability)
- Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing)
- Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts
- Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
2026-08-20 19:22:20 -03:00
parent 6e3d57619b
commit 6a45368cb0
85 changed files with 18345 additions and 3897 deletions
+30 -26
View File
@@ -3,9 +3,9 @@
from __future__ import annotations
import json
from dataclasses import dataclass, field, asdict
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional
from typing import Any
class DecisionCategory(str, Enum):
@@ -29,20 +29,20 @@ class RelatedEntity:
name: str
relation_type: str
weight: float
aliases: List[str] = field(default_factory=list)
aliases: list[str] = field(default_factory=list)
scope: str = "general"
confidence: float = 1.0
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> RelatedEntity:
def from_dict(cls, data: dict[str, Any]) -> RelatedEntity:
if not isinstance(data, dict):
raise ValueError("Related entity must be a JSON object")
required = ["entity_id", "name", "relation_type", "weight"]
for req in required:
if req not in data or data[req] is None:
raise ValueError(f"Missing required field in related entity: '{req}'")
return cls(
entity_id=str(data["entity_id"]),
name=str(data["name"]),
@@ -58,36 +58,36 @@ class RelatedEntity:
class ECPSnapshot:
target_entity_id: str
target_name: str
aliases: List[str]
aliases: list[str]
domain: str
anchors: List[str]
negative_anchors: List[str] = field(default_factory=list)
anchors: list[str]
negative_anchors: list[str] = field(default_factory=list)
graph_version: str = "1.0.0"
related_entities: List[RelatedEntity] = field(default_factory=list)
related_entities: list[RelatedEntity] = field(default_factory=list)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> ECPSnapshot:
def from_dict(cls, data: dict[str, Any]) -> ECPSnapshot:
if not isinstance(data, dict):
raise ValueError("ECP Snapshot payload must be a JSON object")
required_fields = ["target_entity_id", "target_name", "aliases", "domain", "anchors"]
for field_name in required_fields:
if field_name not in data or data[field_name] is None:
raise ValueError(f"Missing required field in ECP Snapshot: '{field_name}'")
if not isinstance(data["aliases"], list):
raise ValueError("Field 'aliases' must be a list of strings")
if not isinstance(data["anchors"], list):
raise ValueError("Field 'anchors' must be a list of strings")
neg_anchors = data.get("negative_anchors", [])
if neg_anchors is not None and not isinstance(neg_anchors, list):
raise ValueError("Field 'negative_anchors' must be a list of strings if provided")
related_data = data.get("related_entities", [])
if related_data is not None and not isinstance(related_data, list):
raise ValueError("Field 'related_entities' must be a list if provided")
related_objs = [RelatedEntity.from_dict(item) for item in (related_data or [])]
return cls(
@@ -124,16 +124,18 @@ class ClassificationResult:
is_inherent: bool
confidence: float
detected_language: str
matched_anchors: List[str] = field(default_factory=list)
negative_matches: List[str] = field(default_factory=list)
graph_matches: List[Dict[str, Any]] = field(default_factory=list)
evidence: List[str] = field(default_factory=list)
matched_anchors: list[str] = field(default_factory=list)
negative_matches: list[str] = field(default_factory=list)
graph_matches: list[dict[str, Any]] = field(default_factory=list)
evidence: list[str] = field(default_factory=list)
rationale: str = ""
warnings: List[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
return {
"decision": self.decision.value if isinstance(self.decision, DecisionCategory) else str(self.decision),
"decision": self.decision.value
if isinstance(self.decision, DecisionCategory)
else str(self.decision),
"is_inherent": bool(self.is_inherent),
"confidence": round(float(self.confidence), 4),
"detected_language": str(self.detected_language),
@@ -153,11 +155,13 @@ class ClassificationResult:
class ClassificationError:
error_code: ErrorCode
message: str
details: Dict[str, Any] = field(default_factory=dict)
details: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
def to_dict(self) -> dict[str, Any]:
return {
"error_code": self.error_code.value if isinstance(self.error_code, ErrorCode) else str(self.error_code),
"error_code": self.error_code.value
if isinstance(self.error_code, ErrorCode)
else str(self.error_code),
"message": str(self.message),
"details": dict(self.details),
}