feat(extractor): implement multi-engine article content extractor

- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability)
- Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing)
- Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts
- Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
2026-08-20 19:22:20 -03:00
parent 6e3d57619b
commit 6a45368cb0
85 changed files with 18345 additions and 3897 deletions
+87 -3
View File
@@ -420,9 +420,9 @@
"semantic_hash": ""
},
"requirements.txt": {
"mtime": 1787236111.876247,
"seen": 1787236262.0105975,
"ast_hash": "9fb7e3f8ecb04ff30d650c772f509557",
"mtime": 1787240110.13472,
"seen": 1787240516.5993943,
"ast_hash": "f5d99630fa9c93c5fbaa44814f107e70",
"semantic_hash": ""
},
"tests/fixtures/benchmark_24/de/contextual.md": {
@@ -652,5 +652,89 @@
"seen": 1787235020.6850297,
"ast_hash": "627a6c953b250083ebe76ee74d605bb5",
"semantic_hash": ""
},
"README.md": {
"mtime": 1787240502.4633865,
"seen": 1787240516.5990582,
"ast_hash": "a075f140e2dd8383bec3991f92b32b26",
"semantic_hash": ""
},
"scripts/extract_article_contents.py": {
"mtime": 1787241061.3097162,
"seen": 1787241108.1811028,
"ast_hash": "08cd37738da1500b6912ffa1fb2fc0cc",
"semantic_hash": ""
},
"tests/test_extract_article_contents.py": {
"mtime": 1787240910.5980105,
"seen": 1787241108.1826632,
"ast_hash": "98cf3b30fa6ce044d943eff381eaf8a7",
"semantic_hash": ""
},
"docs/prd_extrator_artigos_nlp.md": {
"mtime": 1787237774.0092583,
"seen": 1787240516.5991437,
"ast_hash": "c90f25764acefb91f1160feaf497fb33",
"semantic_hash": ""
},
"specs/003-article-content-extractor/checklists/extraction.md": {
"mtime": 1787239873.797626,
"seen": 1787240516.6008725,
"ast_hash": "3c78fb563727a7f6f23240dcaad3c010",
"semantic_hash": ""
},
"specs/003-article-content-extractor/checklists/requirements.md": {
"mtime": 1787237950.8140705,
"seen": 1787240516.600874,
"ast_hash": "16b2bf7a7910722a7aa6c57cdcc31f1c",
"semantic_hash": ""
},
"specs/003-article-content-extractor/contracts/cli-contract.md": {
"mtime": 1787239569.0560138,
"seen": 1787240516.6008754,
"ast_hash": "9326b1b3c124ad41636c3fae5c9de8b5",
"semantic_hash": ""
},
"specs/003-article-content-extractor/contracts/json-schema.md": {
"mtime": 1787239578.9314032,
"seen": 1787240516.6008763,
"ast_hash": "93f1da07b16c61de15dc93071434d145",
"semantic_hash": ""
},
"specs/003-article-content-extractor/data-model.md": {
"mtime": 1787239558.2362826,
"seen": 1787240516.6008773,
"ast_hash": "03ed32de948199c9a498c486e7bdafbd",
"semantic_hash": ""
},
"specs/003-article-content-extractor/plan.md": {
"mtime": 1787239601.395844,
"seen": 1787240516.6008782,
"ast_hash": "406c3ebad1d32984c1b98f8a9b7ae9f1",
"semantic_hash": ""
},
"specs/003-article-content-extractor/quickstart.md": {
"mtime": 1787239589.9896657,
"seen": 1787240516.6008794,
"ast_hash": "7cac454150d33dad67f683367fca749f",
"semantic_hash": ""
},
"specs/003-article-content-extractor/research.md": {
"mtime": 1787239545.8330145,
"seen": 1787240516.6008804,
"ast_hash": "beef81df32e2dc16798ece67726daa95",
"semantic_hash": ""
},
"specs/003-article-content-extractor/spec.md": {
"mtime": 1787239197.9064422,
"seen": 1787240516.6008813,
"ast_hash": "9fd196b7ba4d9ba7adb2cad25d094764",
"semantic_hash": ""
},
"specs/003-article-content-extractor/tasks.md": {
"mtime": 1787240464.6520643,
"seen": 1787240516.6008823,
"ast_hash": "d95c4763728496703cd89590288a2bed",
"semantic_hash": ""
}
}