mirror of
https://github.com/LearningCircuit/local-deep-research.git
synced 2026-06-15 19:46:56 +03:00
2608c5853d
PUNCHLIST Tier 1 flagged test_configure_openrouter_api_key as H1+H4: The test patches SettingsManager.set_setting and get_setting to route through mock_session._set_setting / _get_setting (which both operate on a shared local dict). It then calls set_setting and asserts get_setting returns the same value. Since both set/get are routed through the same mock dict, the roundtrip is guaranteed to work regardless of what the real SettingsManager does. The test would pass even if the real SettingsManager were broken or didn't exist. Real SettingsManager set/get behavior is covered by direct settings tests; this mock-roundtrip "integration" test adds no signal.
464 lines
19 KiB
Python
464 lines
19 KiB
Python
"""
|
|
Integration tests for LLM provider configuration and execution.
|
|
Tests the full flow from API key configuration to research execution.
|
|
"""
|
|
|
|
import pytest
|
|
from unittest.mock import Mock, patch
|
|
from langchain_core.language_models import BaseChatModel
|
|
from local_deep_research.settings import SettingsManager
|
|
|
|
|
|
class TestLLMProviderIntegration:
|
|
"""Test complete LLM provider integration scenarios."""
|
|
|
|
@pytest.fixture
|
|
def settings_dict(self):
|
|
"""Create a dictionary of settings."""
|
|
return {
|
|
"llm.provider": "openai_endpoint",
|
|
"llm.model": "claude-3-sonnet",
|
|
"llm.temperature": 0.7,
|
|
"llm.openai_endpoint.api_key": "sk-openrouter-test-key",
|
|
"llm.openai_endpoint.url": "https://openrouter.ai/api/v1",
|
|
"llm.openai_endpoint.model": "claude-3-sonnet",
|
|
"llm.supports_max_tokens": True,
|
|
"llm.max_tokens": 4096,
|
|
"llm.context_window_unrestricted": True,
|
|
"llm.openai.api_key": None,
|
|
"llm.openai.api_base": None,
|
|
"llm.openai.organization": None,
|
|
"llm.anthropic.api_key": None,
|
|
"llm.streaming": False,
|
|
"llm.max_retries": None,
|
|
"llm.request_timeout": None,
|
|
"llm.ollama.url": "http://localhost:11434",
|
|
"llm.lmstudio.url": "http://localhost:1234",
|
|
"llm.llamacpp.url": "http://localhost:8080/v1",
|
|
"app.lock_settings": False,
|
|
"rate_limiting.llm_enabled": False,
|
|
"search.tool": "searxng",
|
|
"search.iterations": 5,
|
|
"search.questions_per_iteration": 3,
|
|
"search.max_results": 10,
|
|
"search.enable_direct_summary": True,
|
|
"search.enable_search_engine": True,
|
|
"search.additional_results": 3,
|
|
"search.timeout": 15,
|
|
"search.region": "en-US",
|
|
"search.proxy": None,
|
|
"search.validate_sources": True,
|
|
"search.enable_think_tags": False,
|
|
"search.require_all_sources": False,
|
|
"search.max_backoff_time": 300,
|
|
"search.time_period": "all",
|
|
"search.safe_search": True,
|
|
"search.snippets_only": True,
|
|
"search.search_language": "en",
|
|
"search.max_filtered_results": 10,
|
|
"search.smart_search.use_query_improvement": True,
|
|
"search.smart_search.use_document_relevance": True,
|
|
"search.smart_search.use_answer_extraction": True,
|
|
"search.smart_search.use_semantic_cache": True,
|
|
"search.smart_search.use_result_reranking": True,
|
|
"search.smart_search.use_query_suggestion": True,
|
|
"llm.local_context_window_size": 4096,
|
|
"llm.context_window_size": 128000,
|
|
}
|
|
|
|
@pytest.fixture
|
|
def mock_session(self, settings_dict):
|
|
"""Create a mock database session with settings."""
|
|
session = Mock()
|
|
|
|
# Store settings that can be modified
|
|
current_settings = settings_dict.copy()
|
|
|
|
# Helper to get settings
|
|
def get_setting_mock(key, default=None, check_env=True):
|
|
return current_settings.get(key, default)
|
|
|
|
# Helper to set settings
|
|
def set_setting_mock(key, value, commit=True):
|
|
current_settings[key] = value
|
|
return True
|
|
|
|
# Helper to get all settings
|
|
def get_all_settings_mock():
|
|
# Return in the format expected by the API
|
|
result = {}
|
|
for key, value in current_settings.items():
|
|
result[key] = {
|
|
"value": value,
|
|
"type": "STRING",
|
|
"name": key,
|
|
"description": f"Setting for {key}",
|
|
"category": key.split(".")[0],
|
|
"ui_element": "text",
|
|
"visible": True,
|
|
"editable": True,
|
|
}
|
|
return result
|
|
|
|
# Store these helpers on the session for patching
|
|
session._get_setting = get_setting_mock
|
|
session._set_setting = set_setting_mock
|
|
session._get_all_settings = get_all_settings_mock
|
|
session._current_settings = current_settings
|
|
|
|
return session
|
|
|
|
def test_research_with_configured_openrouter(self, mock_session):
|
|
"""Test running research with configured OpenRouter API."""
|
|
from local_deep_research.api.research_functions import quick_summary
|
|
|
|
# Create settings manager with patched methods
|
|
with patch.object(
|
|
SettingsManager,
|
|
"get_all_settings",
|
|
side_effect=mock_session._get_all_settings,
|
|
):
|
|
settings_manager = SettingsManager(None)
|
|
settings_snapshot = settings_manager.get_all_settings()
|
|
|
|
# Verify settings are correct - get_all_settings returns nested dicts
|
|
assert (
|
|
settings_snapshot["llm.provider"]["value"] == "openai_endpoint"
|
|
)
|
|
assert (
|
|
settings_snapshot["llm.openai_endpoint.api_key"]["value"]
|
|
== "sk-openrouter-test-key"
|
|
)
|
|
|
|
# Patch ChatOpenAI in both locations: the auto-discovered provider
|
|
# path (openai_base) and the hardcoded fallback (llm_config).
|
|
# Which path runs depends on whether other tests triggered
|
|
# provider auto-registration in this process.
|
|
with patch(
|
|
"local_deep_research.llm.providers.openai_base.ChatOpenAI"
|
|
) as mock_openai:
|
|
with (
|
|
patch(
|
|
"local_deep_research.config.llm_config.ChatOpenAI",
|
|
mock_openai,
|
|
),
|
|
patch(
|
|
"local_deep_research.api.research_functions.AdvancedSearchSystem"
|
|
) as mock_search_system,
|
|
):
|
|
# Setup mock LLM (spec=BaseChatModel so isinstance check passes)
|
|
mock_llm_instance = Mock(spec=BaseChatModel)
|
|
mock_response = Mock()
|
|
mock_response.content = "Based on my research: Test topic is important because..."
|
|
mock_llm_instance.invoke.return_value = mock_response
|
|
mock_openai.return_value = mock_llm_instance
|
|
|
|
# Setup mock search system
|
|
mock_system_instance = Mock()
|
|
mock_system_instance.analyze_topic.return_value = {
|
|
"current_knowledge": "Based on my research: Test topic is important because...",
|
|
"sources": ["https://example.com/test-topic"],
|
|
"all_links_of_system": [
|
|
"https://example.com/test-topic"
|
|
],
|
|
"findings": [],
|
|
"iterations": 2,
|
|
"questions": {},
|
|
}
|
|
mock_search_system.return_value = mock_system_instance
|
|
|
|
# Run research
|
|
result = quick_summary(
|
|
query="Explain test topic",
|
|
iterations=2,
|
|
questions_per_iteration=3,
|
|
settings_snapshot=settings_snapshot,
|
|
)
|
|
|
|
# Verify OpenRouter was configured correctly
|
|
assert mock_openai.called
|
|
call_args = mock_openai.call_args
|
|
assert (
|
|
call_args.kwargs["api_key"] == "sk-openrouter-test-key"
|
|
)
|
|
# The URL kwarg name differs by code path:
|
|
# registry path uses "base_url", direct path uses "openai_api_base"
|
|
url_value = call_args.kwargs.get(
|
|
"base_url", call_args.kwargs.get("openai_api_base")
|
|
)
|
|
assert url_value == "https://openrouter.ai/api/v1"
|
|
assert call_args.kwargs["model"] == "claude-3-sonnet"
|
|
|
|
# Verify research completed
|
|
assert "summary" in result
|
|
assert "sources" in result
|
|
|
|
def test_switching_providers_dynamically(self, mock_session):
|
|
"""Test switching between providers dynamically."""
|
|
from local_deep_research.config.llm_config import get_llm
|
|
|
|
with patch.object(
|
|
SettingsManager,
|
|
"get_setting",
|
|
side_effect=mock_session._get_setting,
|
|
):
|
|
with patch.object(
|
|
SettingsManager,
|
|
"set_setting",
|
|
side_effect=mock_session._set_setting,
|
|
):
|
|
with patch.object(
|
|
SettingsManager,
|
|
"get_all_settings",
|
|
side_effect=mock_session._get_all_settings,
|
|
):
|
|
settings_manager = SettingsManager(None)
|
|
|
|
# Test 1: OpenRouter configuration
|
|
# Patch both the auto-discovered provider path and the
|
|
# hardcoded fallback in llm_config (which path runs
|
|
# depends on whether other tests registered providers).
|
|
settings_manager.set_setting(
|
|
"llm.provider", "openai_endpoint"
|
|
)
|
|
settings_manager.set_setting(
|
|
"llm.openai_endpoint.api_key", "sk-openrouter-key"
|
|
)
|
|
settings_snapshot = settings_manager.get_all_settings()
|
|
|
|
with patch(
|
|
"local_deep_research.llm.providers.openai_base.ChatOpenAI"
|
|
) as mock_openai:
|
|
with patch(
|
|
"local_deep_research.config.llm_config.ChatOpenAI",
|
|
mock_openai,
|
|
):
|
|
mock_openai.return_value = Mock(spec=BaseChatModel)
|
|
get_llm(settings_snapshot=settings_snapshot)
|
|
assert (
|
|
mock_openai.call_args.kwargs["api_key"]
|
|
== "sk-openrouter-key"
|
|
)
|
|
|
|
# Test 2: Switch to OpenAI
|
|
settings_manager.set_setting("llm.provider", "openai")
|
|
settings_manager.set_setting(
|
|
"llm.openai.api_key", "sk-openai-key"
|
|
)
|
|
settings_snapshot = settings_manager.get_all_settings()
|
|
|
|
with patch(
|
|
"local_deep_research.llm.providers.implementations.openai.ChatOpenAI"
|
|
) as mock_openai:
|
|
with patch(
|
|
"local_deep_research.config.llm_config.ChatOpenAI",
|
|
mock_openai,
|
|
):
|
|
mock_openai.return_value = Mock(spec=BaseChatModel)
|
|
get_llm(settings_snapshot=settings_snapshot)
|
|
assert (
|
|
mock_openai.call_args.kwargs["api_key"]
|
|
== "sk-openai-key"
|
|
)
|
|
|
|
# Test 3: Switch to Anthropic
|
|
settings_manager.set_setting("llm.provider", "anthropic")
|
|
settings_manager.set_setting(
|
|
"llm.anthropic.api_key", "sk-anthropic-key"
|
|
)
|
|
settings_snapshot = settings_manager.get_all_settings()
|
|
|
|
with patch(
|
|
"local_deep_research.llm.providers.implementations.anthropic.ChatAnthropic"
|
|
) as mock_anthropic:
|
|
with patch(
|
|
"local_deep_research.config.llm_config.ChatAnthropic",
|
|
mock_anthropic,
|
|
):
|
|
mock_anthropic.return_value = Mock(
|
|
spec=BaseChatModel
|
|
)
|
|
get_llm(settings_snapshot=settings_snapshot)
|
|
assert (
|
|
mock_anthropic.call_args.kwargs[
|
|
"anthropic_api_key"
|
|
]
|
|
== "sk-anthropic-key"
|
|
)
|
|
|
|
def test_api_key_validation_before_research(self, mock_session):
|
|
"""Test that API key presence is validated before starting research."""
|
|
from local_deep_research.config.llm_config import (
|
|
is_openai_endpoint_available,
|
|
)
|
|
|
|
with patch.object(
|
|
SettingsManager,
|
|
"set_setting",
|
|
side_effect=mock_session._set_setting,
|
|
):
|
|
settings_manager = SettingsManager(None)
|
|
|
|
# With API key
|
|
settings_manager.set_setting(
|
|
"llm.openai_endpoint.api_key", "sk-test-key"
|
|
)
|
|
settings_with_key = {
|
|
"llm.openai_endpoint.api_key": {
|
|
"value": "sk-test-key",
|
|
"type": "str",
|
|
}
|
|
}
|
|
assert (
|
|
is_openai_endpoint_available(
|
|
settings_snapshot=settings_with_key
|
|
)
|
|
is True
|
|
)
|
|
|
|
# Without API key
|
|
settings_manager.set_setting("llm.openai_endpoint.api_key", None)
|
|
settings_without_key = {
|
|
"llm.openai_endpoint.api_key": {"value": None, "type": "str"}
|
|
}
|
|
assert (
|
|
is_openai_endpoint_available(
|
|
settings_snapshot=settings_without_key
|
|
)
|
|
is False
|
|
)
|
|
|
|
@pytest.mark.requires_llm
|
|
def test_custom_model_configuration(self, mock_session):
|
|
"""Test configuring custom models with API endpoints."""
|
|
with patch.object(
|
|
SettingsManager,
|
|
"get_setting",
|
|
side_effect=mock_session._get_setting,
|
|
):
|
|
with patch.object(
|
|
SettingsManager,
|
|
"set_setting",
|
|
side_effect=mock_session._set_setting,
|
|
):
|
|
with patch.object(
|
|
SettingsManager,
|
|
"get_all_settings",
|
|
side_effect=mock_session._get_all_settings,
|
|
):
|
|
settings_manager = SettingsManager(None)
|
|
|
|
# Configure for a specific model on OpenRouter
|
|
settings_manager.set_setting(
|
|
"llm.provider", "openai_endpoint"
|
|
)
|
|
settings_manager.set_setting(
|
|
"llm.model", "anthropic/claude-3-opus"
|
|
)
|
|
settings_manager.set_setting(
|
|
"llm.openai_endpoint.api_key", "sk-openrouter-key"
|
|
)
|
|
settings_manager.set_setting(
|
|
"llm.openai_endpoint.url",
|
|
"https://openrouter.ai/api/v1",
|
|
)
|
|
settings_manager.set_setting("llm.temperature", 0.3)
|
|
|
|
settings_snapshot = settings_manager.get_all_settings()
|
|
|
|
with patch(
|
|
"local_deep_research.config.llm_config.ChatOpenAI"
|
|
) as mock_openai:
|
|
mock_openai.return_value = Mock(spec=BaseChatModel)
|
|
|
|
from local_deep_research.config.llm_config import (
|
|
get_llm,
|
|
)
|
|
|
|
get_llm(settings_snapshot=settings_snapshot)
|
|
|
|
# Verify model configuration
|
|
call_args = mock_openai.call_args
|
|
assert (
|
|
call_args.kwargs["model"]
|
|
== "anthropic/claude-3-opus"
|
|
)
|
|
assert call_args.kwargs["temperature"] == 0.3
|
|
assert (
|
|
call_args.kwargs["api_key"] == "sk-openrouter-key"
|
|
)
|
|
assert (
|
|
call_args.kwargs["openai_api_base"]
|
|
== "https://openrouter.ai/api/v1"
|
|
)
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.skip(
|
|
reason=(
|
|
"BenchmarkService.create_benchmark_run inserts into the encrypted "
|
|
"user database, which the unit-test fixture does not stand up. "
|
|
"Run as part of the integration suite with a real DB session."
|
|
)
|
|
)
|
|
def test_benchmark_with_api_configured_llm(self, mock_session):
|
|
"""Test running benchmarks with API-configured LLM."""
|
|
from local_deep_research.benchmarks.web_api.benchmark_service import (
|
|
BenchmarkService,
|
|
)
|
|
|
|
with patch.object(
|
|
SettingsManager,
|
|
"set_setting",
|
|
side_effect=mock_session._set_setting,
|
|
):
|
|
with patch.object(
|
|
SettingsManager,
|
|
"get_all_settings",
|
|
side_effect=mock_session._get_all_settings,
|
|
):
|
|
settings_manager = SettingsManager(None)
|
|
settings_manager.set_setting("llm.provider", "openai_endpoint")
|
|
settings_manager.set_setting(
|
|
"llm.openai_endpoint.api_key", "sk-benchmark-key"
|
|
)
|
|
|
|
benchmark_service = BenchmarkService()
|
|
|
|
# Create benchmark configuration
|
|
search_config = {
|
|
"iterations": 2,
|
|
"questions_per_iteration": 3,
|
|
"search_tool": "searxng",
|
|
"provider": "openai_endpoint",
|
|
"model_name": "claude-3-sonnet",
|
|
}
|
|
|
|
evaluation_config = {
|
|
"provider": "openai_endpoint",
|
|
"model_name": "claude-3-sonnet",
|
|
}
|
|
|
|
datasets_config = {"simpleqa": {"count": 5}}
|
|
|
|
# Test benchmark creation
|
|
with patch.object(
|
|
settings_manager, "get_all_settings"
|
|
) as mock_get_settings:
|
|
mock_get_settings.return_value = (
|
|
settings_manager.get_all_settings()
|
|
)
|
|
|
|
benchmark_id = benchmark_service.create_benchmark_run(
|
|
run_name="Test API Key Benchmark",
|
|
search_config=search_config,
|
|
evaluation_config=evaluation_config,
|
|
datasets_config=datasets_config,
|
|
username="testuser",
|
|
)
|
|
|
|
assert isinstance(benchmark_id, int)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|