andreferraro
6a45368cb0
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
62 lines
1.9 KiB
Python
62 lines
1.9 KiB
Python
"""Unit tests for language detection and text normalization."""
|
|
|
|
from src.language import detect_language, normalize_text
|
|
|
|
|
|
def test_normalize_text():
|
|
assert normalize_text("São Paulo & Petróleo") == "sao paulo & petroleo"
|
|
assert normalize_text(
|
|
"Über große Veränderungen"
|
|
) == "uber grosse veranderungen" or "uber" in normalize_text("Über")
|
|
assert normalize_text("Crème brûlée") == "creme brulee"
|
|
|
|
|
|
def test_detect_portuguese():
|
|
text = "A Petrobras anunciou um novo plano de investimentos para a exploração de petróleo na camada pré-sal."
|
|
lang, conf = detect_language(text)
|
|
assert lang == "pt"
|
|
assert conf > 0.5
|
|
|
|
|
|
def test_detect_english():
|
|
text = "Apple announced its new silicon chip with improved machine learning performance and battery life."
|
|
lang, conf = detect_language(text)
|
|
assert lang == "en"
|
|
assert conf > 0.5
|
|
|
|
|
|
def test_detect_spanish():
|
|
text = (
|
|
"La empresa petrolera anunció una nueva inversión en el sector energético durante este año."
|
|
)
|
|
lang, conf = detect_language(text)
|
|
assert lang == "es"
|
|
assert conf > 0.5
|
|
|
|
|
|
def test_detect_german():
|
|
text = "Volkswagen plant eine umfassende Transformation zur Elektromobilität in den kommenden Jahren."
|
|
lang, conf = detect_language(text)
|
|
assert lang == "de"
|
|
assert conf > 0.5
|
|
|
|
|
|
def test_detect_italian():
|
|
text = "La Ferrari ha presentato la nuova vettura da competizione per il campionato mondiale di Formula 1."
|
|
lang, conf = detect_language(text)
|
|
assert lang == "it"
|
|
assert conf > 0.5
|
|
|
|
|
|
def test_detect_french():
|
|
text = "Le groupe TotalEnergies a confirmé ses nouveaux projets de développement dans les énergies renouvelables."
|
|
lang, conf = detect_language(text)
|
|
assert lang == "fr"
|
|
assert conf > 0.5
|
|
|
|
|
|
def test_empty_language():
|
|
lang, conf = detect_language("")
|
|
assert lang == "unknown"
|
|
assert conf == 0.0
|