feat(extractor): add Google News headlines extractor with Foxcape headless and URL resolution
- Add standalone CLI script scripts/extract_google_news.py for Google News RSS scraping - Integrate foxcape in headless mode as primary stealth anti-bot engine - Implement parallel article URL resolution using googlenewsdecoder and ThreadPoolExecutor - Support language and regional locale mapping (-l, --lang, --locale) - Implement real-time progress logging in stderr and --silent flag - Add unit, integration, and live E2E tests in tests/test_extract_google_news.py - Add full SpecKit documentation (specs/002-google-news-extractor/) - Create comprehensive README.md covering both NLP Classifier and Google News Extractor
This commit is contained in:
@@ -300,9 +300,9 @@
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"pyproject.toml": {
|
||||
"mtime": 1787195827.3305018,
|
||||
"seen": 1787196463.0953014,
|
||||
"ast_hash": "f2503e96d08f5a4e41b13352e81709c6",
|
||||
"mtime": 1787234982.768605,
|
||||
"seen": 1787235020.6850271,
|
||||
"ast_hash": "26f5f979658067bf447f375f044a8f21",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"src/__init__.py": {
|
||||
@@ -420,9 +420,9 @@
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"requirements.txt": {
|
||||
"mtime": 1787195817.4782183,
|
||||
"seen": 1787196463.1020837,
|
||||
"ast_hash": "5612073a1e7034d7c25765379baa5da4",
|
||||
"mtime": 1787236111.876247,
|
||||
"seen": 1787236262.0105975,
|
||||
"ast_hash": "9fb7e3f8ecb04ff30d650c772f509557",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"tests/fixtures/benchmark_24/de/contextual.md": {
|
||||
@@ -574,5 +574,89 @@
|
||||
"seen": 1787197159.9881344,
|
||||
"ast_hash": "5e8daf517ee32c261617d96264ef0473",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"scripts/extract_google_news.py": {
|
||||
"mtime": 1787236453.5231855,
|
||||
"seen": 1787236509.1147037,
|
||||
"ast_hash": "4216de490a87378d21dab707a3166679",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"tests/test_extract_google_news.py": {
|
||||
"mtime": 1787236301.513448,
|
||||
"seen": 1787236353.5582018,
|
||||
"ast_hash": "1fc9a108a8abfecf1d37c8642a885e9e",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"docs/googlenews_extractor_guia_completo.md": {
|
||||
"mtime": 1787231605.0830677,
|
||||
"seen": 1787234772.8112028,
|
||||
"ast_hash": "59db8e652000ba6087a00289a0ce0959",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/checklists/readiness.md": {
|
||||
"mtime": 1787234305.932687,
|
||||
"seen": 1787234772.8123577,
|
||||
"ast_hash": "a1200a6959d41e256a73403dd5c4e1f6",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/checklists/requirements.md": {
|
||||
"mtime": 1787232386.6763985,
|
||||
"seen": 1787234772.812359,
|
||||
"ast_hash": "39cd86af76c9f74baa2984418870b677",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/contracts/cli_contract.md": {
|
||||
"mtime": 1787236744.3139226,
|
||||
"seen": 1787236817.4979818,
|
||||
"ast_hash": "76a61ad31df989244edc5e739081bf7b",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/data-model.md": {
|
||||
"mtime": 1787234040.8134317,
|
||||
"seen": 1787234772.8123617,
|
||||
"ast_hash": "e8380c98d2f6418f60ac8a511ecdb637",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/plan.md": {
|
||||
"mtime": 1787236725.9825225,
|
||||
"seen": 1787236817.4983613,
|
||||
"ast_hash": "a787c974414ae6c173d4c39e76113c31",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/quickstart.md": {
|
||||
"mtime": 1787236764.0206513,
|
||||
"seen": 1787236817.4983652,
|
||||
"ast_hash": "ada22d34a0fa8341f88136cf42f055a9",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/research.md": {
|
||||
"mtime": 1787236785.0466182,
|
||||
"seen": 1787236817.498368,
|
||||
"ast_hash": "60837e6c7463c4413911ceca0087374c",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/spec.md": {
|
||||
"mtime": 1787236709.0783155,
|
||||
"seen": 1787236817.4983711,
|
||||
"ast_hash": "dfd963f7ca351c93e09f74af9b11d19e",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/002-google-news-extractor/tasks.md": {
|
||||
"mtime": 1787236802.4808047,
|
||||
"seen": 1787236817.4983742,
|
||||
"ast_hash": "427f24026deaa7c9aca95d99e7587ae6",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"scripts/__init__.py": {
|
||||
"mtime": 1787234973.5782337,
|
||||
"seen": 1787235020.6850297,
|
||||
"ast_hash": "627a6c953b250083ebe76ee74d605bb5",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"README.md": {
|
||||
"mtime": 1787237002.3288481,
|
||||
"seen": 1787237013.7487457,
|
||||
"ast_hash": "1b624e85d8ce1521683cbea2664dd2d2",
|
||||
"semantic_hash": ""
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user