test(e2e): add comprehensive Senior QA E2E text analysis and LLM fallback funnel suite

This commit is contained in:
2026-08-21 11:06:57 -03:00
parent bae144055e
commit a874b98dac
17 changed files with 2260 additions and 815 deletions
+74 -6
View File
@@ -1,19 +1,44 @@
"""Optional LLM fallback adapter (Tier 3).
Disabled by default. Provides fallback interface for boundary disambiguation
without requiring external API keys for core POC execution.
supporting OpenAI and Gemini APIs with automatic .env loading and custom provider functions.
"""
from __future__ import annotations
import json
import os
import urllib.error
import urllib.request
from pathlib import Path
from typing import Callable, Optional
from src.adapters.base import BaseNLPAdapter
from src.models import ClassificationResult, DecisionCategory, ECPSnapshot
def _load_env_file() -> None:
"""Carrega variáveis do arquivo .env na raiz do projeto se existir."""
for parent in [Path.cwd(), Path(__file__).parent.parent.parent]:
env_file = parent / ".env"
if env_file.is_file():
try:
for line in env_file.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, val = line.split("=", 1)
key = key.strip()
val = val.strip().strip("'\"")
if key and key not in os.environ:
os.environ[key] = val
except Exception:
pass
break
_load_env_file()
class LLMFallbackAdapter(BaseNLPAdapter):
"""Optional adapter for LLM fallback boundary disambiguation."""
@@ -23,13 +48,14 @@ class LLMFallbackAdapter(BaseNLPAdapter):
api_key: str | None = None,
provider_fn: Optional[Callable[[str], str]] = None,
) -> None:
self.model_name = model_name
self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
self.model_name = os.environ.get("OPENAI_MODEL", model_name)
self.openai_api_key = api_key or os.environ.get("OPENAI_API_KEY")
self.gemini_api_key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
self.provider_fn = provider_fn
def is_available(self) -> bool:
"""Returns True if an API key or custom provider function is configured."""
return bool(self.api_key or self.provider_fn)
return bool(self.openai_api_key or self.gemini_api_key or self.provider_fn)
def evaluate_similarity(self, text: str, terms: list[str]) -> float:
return 0.0
@@ -135,12 +161,54 @@ Respond ONLY with a valid JSON object matching this schema:
prompt = self.build_prompt(ecp, content_md, initial_result)
# If custom provider function is provided (e.g. for testing or custom runtime)
# 1. Custom provider function (testing or custom runtime)
if self.provider_fn is not None:
raw_response = self.provider_fn(prompt)
return self._parse_llm_response(raw_response, initial_result)
# Stub default for POC when only API key string is present without active SDK
# 2. Real OpenAI execution
if self.openai_api_key:
try:
from openai import OpenAI
client = OpenAI(api_key=self.openai_api_key)
response = client.chat.completions.create(
model=self.model_name,
messages=[{"role": "user", "content": prompt}],
response_format={"type": "json_object"},
temperature=0.0,
)
raw_text = response.choices[0].message.content or ""
return self._parse_llm_response(raw_text, initial_result)
except Exception as e:
initial_result.warnings.append(f"OpenAI fallback invocation error: {e}")
return None
# 3. Real Gemini execution via REST API
if self.gemini_api_key:
try:
gemini_model = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash")
url = f"https://generativelanguage.googleapis.com/v1beta/models/{gemini_model}:generateContent?key={self.gemini_api_key}"
payload = {
"contents": [{"parts": [{"text": prompt}]}],
"generationConfig": {
"responseMimeType": "application/json",
"temperature": 0.0,
},
}
req = urllib.request.Request(
url,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read().decode("utf-8"))
raw_text = data["candidates"][0]["content"]["parts"][0]["text"]
return self._parse_llm_response(raw_text, initial_result)
except Exception as e:
initial_result.warnings.append(f"Gemini fallback invocation error: {e}")
return None
return None
def _parse_llm_response(