feat(extractor): implement multi-engine article content extractor

- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability)
- Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing)
- Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts
- Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
2026-08-20 19:22:20 -03:00
parent 6e3d57619b
commit 6a45368cb0
85 changed files with 18345 additions and 3897 deletions
+5 -15
View File
@@ -140,9 +140,7 @@ def test_resolve_article_url_fallback():
direct_url = "https://www.globo.com/noticia/123"
assert resolve_article_url(direct_url) == direct_url
with patch(
"scripts.extract_google_news.gnewsdecoder", return_value={"status": False}
):
with patch("scripts.extract_google_news.gnewsdecoder", return_value={"status": False}):
gn_url = "https://news.google.com/rss/articles/fake_token"
assert resolve_article_url(gn_url) == gn_url
@@ -187,9 +185,7 @@ def test_resolve_articles_urls_batch():
def test_extract_google_news_orchestration_mocked(sample_rss_xml: str):
"""Valida a consolidação do ExtractionResult a partir da busca mockada com URLs resolvidas."""
query = SearchQuery(
keyword="inteligência artificial", language="pt", locale="BR", max_pages=1
)
query = SearchQuery(keyword="inteligência artificial", language="pt", locale="BR", max_pages=1)
with (
patch(
@@ -221,13 +217,9 @@ def test_cli_execution_stdout(sample_rss_xml: str, capsys: pytest.CaptureFixture
"scripts.extract_google_news._fetch_rss_content",
return_value=sample_rss_xml,
),
patch(
"scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u
),
patch("scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u),
):
exit_code = main(
["--query", "inteligencia artificial", "--lang", "pt", "--pretty"]
)
exit_code = main(["--query", "inteligencia artificial", "--lang", "pt", "--pretty"])
assert exit_code == 0
captured = capsys.readouterr()
@@ -248,9 +240,7 @@ def test_cli_execution_file_output(sample_rss_xml: str, tmp_path: Path):
"scripts.extract_google_news._fetch_rss_content",
return_value=sample_rss_xml,
),
patch(
"scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u
),
patch("scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u),
):
exit_code = main(["-q", "IA", "-p", "1", "-o", str(out_file)])
assert exit_code == 0