feat(extractor): implement multi-engine article content extractor
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
@@ -140,9 +140,7 @@ def test_resolve_article_url_fallback():
|
||||
direct_url = "https://www.globo.com/noticia/123"
|
||||
assert resolve_article_url(direct_url) == direct_url
|
||||
|
||||
with patch(
|
||||
"scripts.extract_google_news.gnewsdecoder", return_value={"status": False}
|
||||
):
|
||||
with patch("scripts.extract_google_news.gnewsdecoder", return_value={"status": False}):
|
||||
gn_url = "https://news.google.com/rss/articles/fake_token"
|
||||
assert resolve_article_url(gn_url) == gn_url
|
||||
|
||||
@@ -187,9 +185,7 @@ def test_resolve_articles_urls_batch():
|
||||
|
||||
def test_extract_google_news_orchestration_mocked(sample_rss_xml: str):
|
||||
"""Valida a consolidação do ExtractionResult a partir da busca mockada com URLs resolvidas."""
|
||||
query = SearchQuery(
|
||||
keyword="inteligência artificial", language="pt", locale="BR", max_pages=1
|
||||
)
|
||||
query = SearchQuery(keyword="inteligência artificial", language="pt", locale="BR", max_pages=1)
|
||||
|
||||
with (
|
||||
patch(
|
||||
@@ -221,13 +217,9 @@ def test_cli_execution_stdout(sample_rss_xml: str, capsys: pytest.CaptureFixture
|
||||
"scripts.extract_google_news._fetch_rss_content",
|
||||
return_value=sample_rss_xml,
|
||||
),
|
||||
patch(
|
||||
"scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u
|
||||
),
|
||||
patch("scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u),
|
||||
):
|
||||
exit_code = main(
|
||||
["--query", "inteligencia artificial", "--lang", "pt", "--pretty"]
|
||||
)
|
||||
exit_code = main(["--query", "inteligencia artificial", "--lang", "pt", "--pretty"])
|
||||
assert exit_code == 0
|
||||
|
||||
captured = capsys.readouterr()
|
||||
@@ -248,9 +240,7 @@ def test_cli_execution_file_output(sample_rss_xml: str, tmp_path: Path):
|
||||
"scripts.extract_google_news._fetch_rss_content",
|
||||
return_value=sample_rss_xml,
|
||||
),
|
||||
patch(
|
||||
"scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u
|
||||
),
|
||||
patch("scripts.extract_google_news.resolve_article_url", side_effect=lambda u: u),
|
||||
):
|
||||
exit_code = main(["-q", "IA", "-p", "1", "-o", str(out_file)])
|
||||
assert exit_code == 0
|
||||
|
||||
Reference in New Issue
Block a user