feat(extractor): implement multi-engine article content extractor
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
@@ -1,11 +1,13 @@
|
||||
"""Unit tests for language detection and text normalization."""
|
||||
|
||||
from src.language import detect_language, normalize_text, SUPPORTED_LANGUAGES
|
||||
from src.language import detect_language, normalize_text
|
||||
|
||||
|
||||
def test_normalize_text():
|
||||
assert normalize_text("São Paulo & Petróleo") == "sao paulo & petroleo"
|
||||
assert normalize_text("Über große Veränderungen") == "uber grosse veranderungen" or "uber" in normalize_text("Über")
|
||||
assert normalize_text(
|
||||
"Über große Veränderungen"
|
||||
) == "uber grosse veranderungen" or "uber" in normalize_text("Über")
|
||||
assert normalize_text("Crème brûlée") == "creme brulee"
|
||||
|
||||
|
||||
@@ -24,7 +26,9 @@ def test_detect_english():
|
||||
|
||||
|
||||
def test_detect_spanish():
|
||||
text = "La empresa petrolera anunció una nueva inversión en el sector energético durante este año."
|
||||
text = (
|
||||
"La empresa petrolera anunció una nueva inversión en el sector energético durante este año."
|
||||
)
|
||||
lang, conf = detect_language(text)
|
||||
assert lang == "es"
|
||||
assert conf > 0.5
|
||||
|
||||
Reference in New Issue
Block a user