andreferraro/TextNLPClassifierApp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(extractor): implement multi-engine article content extractor

Browse Source 
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability)
- Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing)
- Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts
- Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
andreferraro
2026-08-20 19:22:20 -03:00
parent 6e3d57619b
commit 6a45368cb0
85 changed files with 18345 additions and 3897 deletions
Download Patch File Download Diff File Expand all files Collapse all files
graphify-out/manifest.json
+141 -63
View File
@@ -294,15 +294,15 @@
"semantic_hash": ""
},
"classify.py": {
"mtime": 1787197110.3753626,
"seen": 1787197159.9828603,
"ast_hash": "d09a35a5e25d42f6ecc537d5b67bef8e",
"mtime": 1787264412.2457643,
"seen": 1787264519.0539427,
"ast_hash": "e89fc4b64bd7606fc466d5338108705b",
"semantic_hash": ""
},
"pyproject.toml": {
"mtime": 1787234982.768605,
"seen": 1787235020.6850271,
"ast_hash": "26f5f979658067bf447f375f044a8f21",
"mtime": 1787264482.8509245,
"seen": 1787264519.0539448,
"ast_hash": "409c1fc0d7bbda6830ccb71c6196cb2f",
"semantic_hash": ""
},
"src/__init__.py": {
@@ -318,45 +318,45 @@
"semantic_hash": ""
},
"src/adapters/base.py": {
"mtime": 1787196396.6199949,
"seen": 1787196463.0953116,
"ast_hash": "40dc6e175c8708467748c1f42a0e064a",
"mtime": 1787264412.2437606,
"seen": 1787264519.0542011,
"ast_hash": "220635d5d74e73f58259069bf5207908",
"semantic_hash": ""
},
"src/adapters/embeddings.py": {
"mtime": 1787196402.7484262,
"seen": 1787196463.0953145,
"ast_hash": "0444823bb05720d4c4ca1662a00b2c01",
"mtime": 1787264437.92723,
"seen": 1787264519.0542023,
"ast_hash": "7beb0ecfb7482c1fd10422da99f69abe",
"semantic_hash": ""
},
"src/adapters/llm.py": {
"mtime": 1787196408.180451,
"seen": 1787196463.0953166,
"ast_hash": "3d95d98625df3fcd343f2fecb78707c5",
"mtime": 1787264412.2437606,
"seen": 1787264519.0542035,
"ast_hash": "ba2990328f5b25ac6cdfc3b57acc9d39",
"semantic_hash": ""
},
"src/classifier.py": {
"mtime": 1787197085.0250447,
"seen": 1787197159.985785,
"ast_hash": "a2e70f968109d213fdab0ae81ac819ff",
"mtime": 1787264459.052089,
"seen": 1787264519.0542045,
"ast_hash": "b8fb440374ecd66bb8bf67c70c19ed1f",
"semantic_hash": ""
},
"src/language.py": {
"mtime": 1787196384.5545745,
"seen": 1787196463.0953217,
"ast_hash": "985619013e58a7f55af2e955817f223c",
"mtime": 1787264437.9312274,
"seen": 1787264519.0542057,
"ast_hash": "cbaf52272ebb05372e34a05cd201c270",
"semantic_hash": ""
},
"src/models.py": {
"mtime": 1787195855.3266282,
"seen": 1787196463.0953236,
"ast_hash": "ce73bfe85eb54f907e2052c80636ecaf",
"mtime": 1787264437.9302285,
"seen": 1787264519.054207,
"ast_hash": "16e604c14f7d66634dc9e2ed7959dd7c",
"semantic_hash": ""
},
"src/parser.py": {
"mtime": 1787195873.9119804,
"seen": 1787196463.0953257,
"ast_hash": "0a1d64ee7088b266125af071f85f3826",
"mtime": 1787264437.9312274,
"seen": 1787264519.054208,
"ast_hash": "d43fca3f063534e3626f86f36a97a2cc",
"semantic_hash": ""
},
"tests/__init__.py": {
@@ -366,39 +366,39 @@
"semantic_hash": ""
},
"tests/test_adapters.py": {
"mtime": 1787196418.6328156,
"seen": 1787196463.09533,
"ast_hash": "5caf95327ea030dbcc37a99bd9052ebd",
"mtime": 1787264437.9292288,
"seen": 1787264519.0543096,
"ast_hash": "7ed9ba6f62fef404bfc28a2e16ada647",
"semantic_hash": ""
},
"tests/test_benchmark_24.py": {
"mtime": 1787196325.5409796,
"seen": 1787196463.0953324,
"ast_hash": "487d30d6de2dee529fa91b86a1d62c0a",
"mtime": 1787264437.9302285,
"seen": 1787264519.0543122,
"ast_hash": "052a51561002be035de1a79455256b67",
"semantic_hash": ""
},
"tests/test_classifier.py": {
"mtime": 1787196364.7195728,
"seen": 1787196463.0953343,
"ast_hash": "4350a5b11a08d12ccb85a3f332d71e87",
"mtime": 1787264437.92723,
"seen": 1787264519.0543132,
"ast_hash": "14da0c7a7d4c08762437ca777c086dea",
"semantic_hash": ""
},
"tests/test_cli.py": {
"mtime": 1787195939.567137,
"seen": 1787196463.0953364,
"ast_hash": "90b0131d43cfc64a76e499c7a65d95de",
"mtime": 1787264437.9292288,
"seen": 1787264519.0543146,
"ast_hash": "b6f57994937363f8a1b0f2f66ea53d3a",
"semantic_hash": ""
},
"tests/test_language.py": {
"mtime": 1787195891.3329668,
"seen": 1787196463.0953388,
"ast_hash": "66191b43fcd7aef47c07cd20d5a8f03c",
"mtime": 1787264437.9292288,
"seen": 1787264519.0543184,
"ast_hash": "f1a09702410864434274f1b347769742",
"semantic_hash": ""
},
"tests/test_models.py": {
"mtime": 1787195886.5009987,
"seen": 1787196463.095341,
"ast_hash": "1fd725c8ca0f52e52f56c7223e9574d0",
"mtime": 1787264437.9302285,
"seen": 1787264519.0543194,
"ast_hash": "7ec72b612bf758be8c9a3617ce18b132",
"semantic_hash": ""
},
"examples/content_northvolt_de.md": {
@@ -420,9 +420,9 @@
"semantic_hash": ""
},
"requirements.txt": {
"mtime": 1787236111.876247,
"seen": 1787236262.0105975,
"ast_hash": "9fb7e3f8ecb04ff30d650c772f509557",
"mtime": 1787240110.13472,
"seen": 1787240516.5993943,
"ast_hash": "f5d99630fa9c93c5fbaa44814f107e70",
"semantic_hash": ""
},
"tests/fixtures/benchmark_24/de/contextual.md": {
@@ -570,27 +570,27 @@
"semantic_hash": ""
},
"tests/test_adversarial.py": {
"mtime": 1787197141.2340336,
"seen": 1787197159.9881344,
"ast_hash": "5e8daf517ee32c261617d96264ef0473",
"mtime": 1787264437.9242287,
"seen": 1787264519.0543108,
"ast_hash": "d353633ce31a624a8dcda790efec4d5e",
"semantic_hash": ""
},
"scripts/extract_google_news.py": {
"mtime": 1787236453.5231855,
"seen": 1787236509.1147037,
"ast_hash": "4216de490a87378d21dab707a3166679",
"mtime": 1787264437.92723,
"seen": 1787264519.0540311,
"ast_hash": "97d8a193901d5e2edf6359fad256b7ef",
"semantic_hash": ""
},
"tests/test_extract_google_news.py": {
"mtime": 1787236301.513448,
"seen": 1787236353.5582018,
"ast_hash": "1fc9a108a8abfecf1d37c8642a885e9e",
"mtime": 1787264437.9312274,
"seen": 1787264519.054317,
"ast_hash": "7561ad8fedcf1bc6cc8ed1e86a345532",
"semantic_hash": ""
},
"docs/googlenews_extractor_guia_completo.md": {
"mtime": 1787231605.0830677,
"seen": 1787234772.8112028,
"ast_hash": "59db8e652000ba6087a00289a0ce0959",
"mtime": 1787264437.9282286,
"seen": 1787264519.0578,
"ast_hash": "d7c3c64e2009dded496a08c7c91f63b9",
"semantic_hash": ""
},
"specs/002-google-news-extractor/checklists/readiness.md": {
@@ -654,9 +654,87 @@
"semantic_hash": ""
},
"README.md": {
"mtime": 1787237002.3288481,
"seen": 1787237013.7487457,
"ast_hash": "1b624e85d8ce1521683cbea2664dd2d2",
"mtime": 1787264334.4498444,
"seen": 1787264519.0577974,
"ast_hash": "196db110cfde06179fb750fc75afb6fe",
"semantic_hash": ""
},
"scripts/extract_article_contents.py": {
"mtime": 1787264465.3415215,
"seen": 1787264519.0540295,
"ast_hash": "2bea6b2a048526cb49d95fb88a001600",
"semantic_hash": ""
},
"tests/test_extract_article_contents.py": {
"mtime": 1787264437.9292288,
"seen": 1787264519.0543158,
"ast_hash": "56aee72480f05c714865791a935d539c",
"semantic_hash": ""
},
"docs/prd_extrator_artigos_nlp.md": {
"mtime": 1787237774.0092583,
"seen": 1787240516.5991437,
"ast_hash": "c90f25764acefb91f1160feaf497fb33",
"semantic_hash": ""
},
"specs/003-article-content-extractor/checklists/extraction.md": {
"mtime": 1787239873.797626,
"seen": 1787240516.6008725,
"ast_hash": "3c78fb563727a7f6f23240dcaad3c010",
"semantic_hash": ""
},
"specs/003-article-content-extractor/checklists/requirements.md": {
"mtime": 1787237950.8140705,
"seen": 1787240516.600874,
"ast_hash": "16b2bf7a7910722a7aa6c57cdcc31f1c",
"semantic_hash": ""
},
"specs/003-article-content-extractor/contracts/cli-contract.md": {
"mtime": 1787239569.0560138,
"seen": 1787240516.6008754,
"ast_hash": "9326b1b3c124ad41636c3fae5c9de8b5",
"semantic_hash": ""
},
"specs/003-article-content-extractor/contracts/json-schema.md": {
"mtime": 1787239578.9314032,
"seen": 1787240516.6008763,
"ast_hash": "93f1da07b16c61de15dc93071434d145",
"semantic_hash": ""
},
"specs/003-article-content-extractor/data-model.md": {
"mtime": 1787239558.2362826,
"seen": 1787240516.6008773,
"ast_hash": "03ed32de948199c9a498c486e7bdafbd",
"semantic_hash": ""
},
"specs/003-article-content-extractor/plan.md": {
"mtime": 1787264323.707266,
"seen": 1787264519.0605621,
"ast_hash": "b2232491829a303a49e749c75f3adca2",
"semantic_hash": ""
},
"specs/003-article-content-extractor/quickstart.md": {
"mtime": 1787239589.9896657,
"seen": 1787240516.6008794,
"ast_hash": "7cac454150d33dad67f683367fca749f",
"semantic_hash": ""
},
"specs/003-article-content-extractor/research.md": {
"mtime": 1787239545.8330145,
"seen": 1787240516.6008804,
"ast_hash": "beef81df32e2dc16798ece67726daa95",
"semantic_hash": ""
},
"specs/003-article-content-extractor/spec.md": {
"mtime": 1787264319.2883234,
"seen": 1787264519.060745,
"ast_hash": "187b1307e28a02f0ac44895b5f9cca55",
"semantic_hash": ""
},
"specs/003-article-content-extractor/tasks.md": {
"mtime": 1787240464.6520643,
"seen": 1787240516.6008823,
"ast_hash": "d95c4763728496703cd89590288a2bed",
"semantic_hash": ""
}
}