feat(extractor): implement multi-engine article content extractor
- Added scripts/extract_article_contents.py for batch scraping with stealth Foxcape and triple extraction (Trafilatura, Newspaper4k, Readability) - Created unit, integration, and E2E test suite in tests/test_extract_article_contents.py (90/90 passing) - Updated specs/003-article-content-extractor and README.md with usage documentation and CLI contracts - Passed ruff linting/formatting and mypy type checking cleanly
This commit is contained in:
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"nodes": [{"id": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_md", "label": "json-schema.md", "file_type": "document", "node_kind": "page", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L1"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_json_schema_contract_article_content_multi_engine_extractor", "label": "JSON Schema Contract: Article Content Multi-Engine Extractor", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L1"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_1_schema_de_entrada_input_json", "label": "1. Schema de Entrada (Input JSON)", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L3"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_2_schema_de_sa\u00edda_output_json", "label": "2. Schema de Sa\u00edda (Output JSON)", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L27"}], "edges": [{"source": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_md", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_json_schema_contract_article_content_multi_engine_extractor", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L1", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_json_schema_contract_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_1_schema_de_entrada_input_json", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L3", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_json_schema_contract_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_json_schema_2_schema_de_sa\u00edda_output_json", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/json-schema.md", "source_location": "L27", "weight": 1.0}], "input_tokens": 0, "output_tokens": 0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"nodes": [{"id": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_md", "label": "requirements.md", "file_type": "document", "node_kind": "page", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L1"}, {"id": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_specification_quality_checklist_article_content_multi_engine_extractor", "label": "Specification Quality Checklist: Article Content Multi-Engine Extractor", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L1"}, {"id": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_content_quality", "label": "Content Quality", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L7"}, {"id": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_requirement_completeness", "label": "Requirement Completeness", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L14"}, {"id": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_feature_readiness", "label": "Feature Readiness", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L25"}, {"id": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_notes", "label": "Notes", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L32"}], "edges": [{"source": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_md", "target": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_specification_quality_checklist_article_content_multi_engine_extractor", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L1", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_md", "target": "$graphify-root$_specs_003_article_content_extractor_spec_md", "relation": "references", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L5", "weight": 1.0, "target_file": "$graphify-root$/specs/003-article-content-extractor/spec.md"}, {"source": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_specification_quality_checklist_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_content_quality", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L7", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_specification_quality_checklist_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_requirement_completeness", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L14", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_specification_quality_checklist_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_feature_readiness", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L25", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_specification_quality_checklist_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_checklists_requirements_notes", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/checklists/requirements.md", "source_location": "L32", "weight": 1.0}], "input_tokens": 0, "output_tokens": 0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"nodes": [{"id": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_md", "label": "cli-contract.md", "file_type": "document", "node_kind": "page", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L1"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_cli_contract_article_content_multi_engine_extractor", "label": "CLI Contract: Article Content Multi-Engine Extractor", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L1"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_1_comando_de_execu\u00e7\u00e3o", "label": "1. Comando de Execu\u00e7\u00e3o", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L3"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_2_argumentos_e_flags", "label": "2. Argumentos e Flags", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L11"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_3_c\u00f3digos_de_sa\u00edda_exit_codes", "label": "3. C\u00f3digos de Sa\u00edda (Exit Codes)", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L25"}, {"id": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_4_comportamento_de_streams_i_o", "label": "4. Comportamento de Streams (I/O)", "file_type": "document", "node_kind": "heading", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L35"}], "edges": [{"source": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_md", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_cli_contract_article_content_multi_engine_extractor", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L1", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_cli_contract_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_1_comando_de_execu\u00e7\u00e3o", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L3", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_cli_contract_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_2_argumentos_e_flags", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L11", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_cli_contract_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_3_c\u00f3digos_de_sa\u00edda_exit_codes", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L25", "weight": 1.0}, {"source": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_cli_contract_article_content_multi_engine_extractor", "target": "$graphify-root$_specs_003_article_content_extractor_contracts_cli_contract_4_comportamento_de_streams_i_o", "relation": "contains", "confidence": "EXTRACTED", "source_file": "specs/003-article-content-extractor/contracts/cli-contract.md", "source_location": "L35", "weight": 1.0}], "input_tokens": 0, "output_tokens": 0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"nodes": [{"id": "pkg_text_nlp_classifier", "label": "text-nlp-classifier", "file_type": "code", "type": "package", "ecosystem": "python", "source_file": "pyproject.toml", "source_location": "L1", "version": "0.1.0"}], "edges": []}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user