feat(extractor): implement multi-engine article content extractor
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
+30
-26
@@ -3,9 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DecisionCategory(str, Enum):
|
||||
@@ -29,20 +29,20 @@ class RelatedEntity:
|
||||
name: str
|
||||
relation_type: str
|
||||
weight: float
|
||||
aliases: List[str] = field(default_factory=list)
|
||||
aliases: list[str] = field(default_factory=list)
|
||||
scope: str = "general"
|
||||
confidence: float = 1.0
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> RelatedEntity:
|
||||
def from_dict(cls, data: dict[str, Any]) -> RelatedEntity:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Related entity must be a JSON object")
|
||||
|
||||
|
||||
required = ["entity_id", "name", "relation_type", "weight"]
|
||||
for req in required:
|
||||
if req not in data or data[req] is None:
|
||||
raise ValueError(f"Missing required field in related entity: '{req}'")
|
||||
|
||||
|
||||
return cls(
|
||||
entity_id=str(data["entity_id"]),
|
||||
name=str(data["name"]),
|
||||
@@ -58,36 +58,36 @@ class RelatedEntity:
|
||||
class ECPSnapshot:
|
||||
target_entity_id: str
|
||||
target_name: str
|
||||
aliases: List[str]
|
||||
aliases: list[str]
|
||||
domain: str
|
||||
anchors: List[str]
|
||||
negative_anchors: List[str] = field(default_factory=list)
|
||||
anchors: list[str]
|
||||
negative_anchors: list[str] = field(default_factory=list)
|
||||
graph_version: str = "1.0.0"
|
||||
related_entities: List[RelatedEntity] = field(default_factory=list)
|
||||
related_entities: list[RelatedEntity] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> ECPSnapshot:
|
||||
def from_dict(cls, data: dict[str, Any]) -> ECPSnapshot:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("ECP Snapshot payload must be a JSON object")
|
||||
|
||||
|
||||
required_fields = ["target_entity_id", "target_name", "aliases", "domain", "anchors"]
|
||||
for field_name in required_fields:
|
||||
if field_name not in data or data[field_name] is None:
|
||||
raise ValueError(f"Missing required field in ECP Snapshot: '{field_name}'")
|
||||
|
||||
|
||||
if not isinstance(data["aliases"], list):
|
||||
raise ValueError("Field 'aliases' must be a list of strings")
|
||||
if not isinstance(data["anchors"], list):
|
||||
raise ValueError("Field 'anchors' must be a list of strings")
|
||||
|
||||
|
||||
neg_anchors = data.get("negative_anchors", [])
|
||||
if neg_anchors is not None and not isinstance(neg_anchors, list):
|
||||
raise ValueError("Field 'negative_anchors' must be a list of strings if provided")
|
||||
|
||||
|
||||
related_data = data.get("related_entities", [])
|
||||
if related_data is not None and not isinstance(related_data, list):
|
||||
raise ValueError("Field 'related_entities' must be a list if provided")
|
||||
|
||||
|
||||
related_objs = [RelatedEntity.from_dict(item) for item in (related_data or [])]
|
||||
|
||||
return cls(
|
||||
@@ -124,16 +124,18 @@ class ClassificationResult:
|
||||
is_inherent: bool
|
||||
confidence: float
|
||||
detected_language: str
|
||||
matched_anchors: List[str] = field(default_factory=list)
|
||||
negative_matches: List[str] = field(default_factory=list)
|
||||
graph_matches: List[Dict[str, Any]] = field(default_factory=list)
|
||||
evidence: List[str] = field(default_factory=list)
|
||||
matched_anchors: list[str] = field(default_factory=list)
|
||||
negative_matches: list[str] = field(default_factory=list)
|
||||
graph_matches: list[dict[str, Any]] = field(default_factory=list)
|
||||
evidence: list[str] = field(default_factory=list)
|
||||
rationale: str = ""
|
||||
warnings: List[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"decision": self.decision.value if isinstance(self.decision, DecisionCategory) else str(self.decision),
|
||||
"decision": self.decision.value
|
||||
if isinstance(self.decision, DecisionCategory)
|
||||
else str(self.decision),
|
||||
"is_inherent": bool(self.is_inherent),
|
||||
"confidence": round(float(self.confidence), 4),
|
||||
"detected_language": str(self.detected_language),
|
||||
@@ -153,11 +155,13 @@ class ClassificationResult:
|
||||
class ClassificationError:
|
||||
error_code: ErrorCode
|
||||
message: str
|
||||
details: Dict[str, Any] = field(default_factory=dict)
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"error_code": self.error_code.value if isinstance(self.error_code, ErrorCode) else str(self.error_code),
|
||||
"error_code": self.error_code.value
|
||||
if isinstance(self.error_code, ErrorCode)
|
||||
else str(self.error_code),
|
||||
"message": str(self.message),
|
||||
"details": dict(self.details),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user