feat(extractor): add Google News headlines extractor with Foxcape headless and URL resolution

- Add standalone CLI script scripts/extract_google_news.py for Google News RSS scraping
- Integrate foxcape in headless mode as primary stealth anti-bot engine
- Implement parallel article URL resolution using googlenewsdecoder and ThreadPoolExecutor
- Support language and regional locale mapping (-l, --lang, --locale)
- Implement real-time progress logging in stderr and --silent flag
- Add unit, integration, and live E2E tests in tests/test_extract_google_news.py
- Add full SpecKit documentation (specs/002-google-news-extractor/)
- Create comprehensive README.md covering both NLP Classifier and Google News Extractor
This commit is contained in:
2026-08-20 11:50:16 -03:00
parent 67cc40f91a
commit 6e3d57619b
59 changed files with 16118 additions and 2160 deletions
+96 -12
View File
@@ -294,15 +294,15 @@
"semantic_hash": ""
},
"classify.py": {
"mtime": 1787195933.7872643,
"seen": 1787196463.095295,
"ast_hash": "5fa90ecfb87bdf15ee99165a3be7c63b",
"mtime": 1787197110.3753626,
"seen": 1787197159.9828603,
"ast_hash": "d09a35a5e25d42f6ecc537d5b67bef8e",
"semantic_hash": ""
},
"pyproject.toml": {
"mtime": 1787195827.3305018,
"seen": 1787196463.0953014,
"ast_hash": "f2503e96d08f5a4e41b13352e81709c6",
"mtime": 1787234982.768605,
"seen": 1787235020.6850271,
"ast_hash": "26f5f979658067bf447f375f044a8f21",
"semantic_hash": ""
},
"src/__init__.py": {
@@ -336,9 +336,9 @@
"semantic_hash": ""
},
"src/classifier.py": {
"mtime": 1787196413.3774571,
"seen": 1787196463.095319,
"ast_hash": "d7bab620ae29badf0d3779fab44552bc",
"mtime": 1787197085.0250447,
"seen": 1787197159.985785,
"ast_hash": "a2e70f968109d213fdab0ae81ac819ff",
"semantic_hash": ""
},
"src/language.py": {
@@ -420,9 +420,9 @@
"semantic_hash": ""
},
"requirements.txt": {
"mtime": 1787195817.4782183,
"seen": 1787196463.1020837,
"ast_hash": "5612073a1e7034d7c25765379baa5da4",
"mtime": 1787236111.876247,
"seen": 1787236262.0105975,
"ast_hash": "9fb7e3f8ecb04ff30d650c772f509557",
"semantic_hash": ""
},
"tests/fixtures/benchmark_24/de/contextual.md": {
@@ -568,5 +568,89 @@
"seen": 1787196463.1031141,
"ast_hash": "e05ab20a5190cfb5b9d41da64d5273cf",
"semantic_hash": ""
},
"tests/test_adversarial.py": {
"mtime": 1787197141.2340336,
"seen": 1787197159.9881344,
"ast_hash": "5e8daf517ee32c261617d96264ef0473",
"semantic_hash": ""
},
"scripts/extract_google_news.py": {
"mtime": 1787236453.5231855,
"seen": 1787236509.1147037,
"ast_hash": "4216de490a87378d21dab707a3166679",
"semantic_hash": ""
},
"tests/test_extract_google_news.py": {
"mtime": 1787236301.513448,
"seen": 1787236353.5582018,
"ast_hash": "1fc9a108a8abfecf1d37c8642a885e9e",
"semantic_hash": ""
},
"docs/googlenews_extractor_guia_completo.md": {
"mtime": 1787231605.0830677,
"seen": 1787234772.8112028,
"ast_hash": "59db8e652000ba6087a00289a0ce0959",
"semantic_hash": ""
},
"specs/002-google-news-extractor/checklists/readiness.md": {
"mtime": 1787234305.932687,
"seen": 1787234772.8123577,
"ast_hash": "a1200a6959d41e256a73403dd5c4e1f6",
"semantic_hash": ""
},
"specs/002-google-news-extractor/checklists/requirements.md": {
"mtime": 1787232386.6763985,
"seen": 1787234772.812359,
"ast_hash": "39cd86af76c9f74baa2984418870b677",
"semantic_hash": ""
},
"specs/002-google-news-extractor/contracts/cli_contract.md": {
"mtime": 1787236744.3139226,
"seen": 1787236817.4979818,
"ast_hash": "76a61ad31df989244edc5e739081bf7b",
"semantic_hash": ""
},
"specs/002-google-news-extractor/data-model.md": {
"mtime": 1787234040.8134317,
"seen": 1787234772.8123617,
"ast_hash": "e8380c98d2f6418f60ac8a511ecdb637",
"semantic_hash": ""
},
"specs/002-google-news-extractor/plan.md": {
"mtime": 1787236725.9825225,
"seen": 1787236817.4983613,
"ast_hash": "a787c974414ae6c173d4c39e76113c31",
"semantic_hash": ""
},
"specs/002-google-news-extractor/quickstart.md": {
"mtime": 1787236764.0206513,
"seen": 1787236817.4983652,
"ast_hash": "ada22d34a0fa8341f88136cf42f055a9",
"semantic_hash": ""
},
"specs/002-google-news-extractor/research.md": {
"mtime": 1787236785.0466182,
"seen": 1787236817.498368,
"ast_hash": "60837e6c7463c4413911ceca0087374c",
"semantic_hash": ""
},
"specs/002-google-news-extractor/spec.md": {
"mtime": 1787236709.0783155,
"seen": 1787236817.4983711,
"ast_hash": "dfd963f7ca351c93e09f74af9b11d19e",
"semantic_hash": ""
},
"specs/002-google-news-extractor/tasks.md": {
"mtime": 1787236802.4808047,
"seen": 1787236817.4983742,
"ast_hash": "427f24026deaa7c9aca95d99e7587ae6",
"semantic_hash": ""
},
"scripts/__init__.py": {
"mtime": 1787234973.5782337,
"seen": 1787235020.6850297,
"ast_hash": "627a6c953b250083ebe76ee74d605bb5",
"semantic_hash": ""
}
}