feat(extractor): implement multi-engine article content extractor
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
@@ -420,9 +420,9 @@
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"requirements.txt": {
|
||||
"mtime": 1787236111.876247,
|
||||
"seen": 1787236262.0105975,
|
||||
"ast_hash": "9fb7e3f8ecb04ff30d650c772f509557",
|
||||
"mtime": 1787240110.13472,
|
||||
"seen": 1787240516.5993943,
|
||||
"ast_hash": "f5d99630fa9c93c5fbaa44814f107e70",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"tests/fixtures/benchmark_24/de/contextual.md": {
|
||||
@@ -652,5 +652,89 @@
|
||||
"seen": 1787235020.6850297,
|
||||
"ast_hash": "627a6c953b250083ebe76ee74d605bb5",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"README.md": {
|
||||
"mtime": 1787240502.4633865,
|
||||
"seen": 1787240516.5990582,
|
||||
"ast_hash": "a075f140e2dd8383bec3991f92b32b26",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"scripts/extract_article_contents.py": {
|
||||
"mtime": 1787241061.3097162,
|
||||
"seen": 1787241108.1811028,
|
||||
"ast_hash": "08cd37738da1500b6912ffa1fb2fc0cc",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"tests/test_extract_article_contents.py": {
|
||||
"mtime": 1787240910.5980105,
|
||||
"seen": 1787241108.1826632,
|
||||
"ast_hash": "98cf3b30fa6ce044d943eff381eaf8a7",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"docs/prd_extrator_artigos_nlp.md": {
|
||||
"mtime": 1787237774.0092583,
|
||||
"seen": 1787240516.5991437,
|
||||
"ast_hash": "c90f25764acefb91f1160feaf497fb33",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/checklists/extraction.md": {
|
||||
"mtime": 1787239873.797626,
|
||||
"seen": 1787240516.6008725,
|
||||
"ast_hash": "3c78fb563727a7f6f23240dcaad3c010",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/checklists/requirements.md": {
|
||||
"mtime": 1787237950.8140705,
|
||||
"seen": 1787240516.600874,
|
||||
"ast_hash": "16b2bf7a7910722a7aa6c57cdcc31f1c",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/contracts/cli-contract.md": {
|
||||
"mtime": 1787239569.0560138,
|
||||
"seen": 1787240516.6008754,
|
||||
"ast_hash": "9326b1b3c124ad41636c3fae5c9de8b5",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/contracts/json-schema.md": {
|
||||
"mtime": 1787239578.9314032,
|
||||
"seen": 1787240516.6008763,
|
||||
"ast_hash": "93f1da07b16c61de15dc93071434d145",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/data-model.md": {
|
||||
"mtime": 1787239558.2362826,
|
||||
"seen": 1787240516.6008773,
|
||||
"ast_hash": "03ed32de948199c9a498c486e7bdafbd",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/plan.md": {
|
||||
"mtime": 1787239601.395844,
|
||||
"seen": 1787240516.6008782,
|
||||
"ast_hash": "406c3ebad1d32984c1b98f8a9b7ae9f1",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/quickstart.md": {
|
||||
"mtime": 1787239589.9896657,
|
||||
"seen": 1787240516.6008794,
|
||||
"ast_hash": "7cac454150d33dad67f683367fca749f",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/research.md": {
|
||||
"mtime": 1787239545.8330145,
|
||||
"seen": 1787240516.6008804,
|
||||
"ast_hash": "beef81df32e2dc16798ece67726daa95",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/spec.md": {
|
||||
"mtime": 1787239197.9064422,
|
||||
"seen": 1787240516.6008813,
|
||||
"ast_hash": "9fd196b7ba4d9ba7adb2cad25d094764",
|
||||
"semantic_hash": ""
|
||||
},
|
||||
"specs/003-article-content-extractor/tasks.md": {
|
||||
"mtime": 1787240464.6520643,
|
||||
"seen": 1787240516.6008823,
|
||||
"ast_hash": "d95c4763728496703cd89590288a2bed",
|
||||
"semantic_hash": ""
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user