mirror of
https://github.com/LearningCircuit/local-deep-research.git
synced 2026-06-16 03:51:07 +03:00
cefdf70a4e
* fix: detect redundant exception args in logger.exception pre-commit hook
The regex-based check only caught f-string interpolation ({e}) but missed
%-style formatting (logger.exception("..%s..", e)), str(e)/repr(e) as
arguments, and {str(e)} inside f-strings. Replaced the regex with an
AST-based check that reliably catches all forms.
* fix: remove redundant exception vars from logger.exception calls
logger.exception() automatically includes the full traceback, so
passing the exception variable (e, exc, err, etc.) is redundant.
Removed 143 instances across 71 files including f-string {e},
%-style formatting, and bare exception arg patterns.
* fix: enhance hook with string concat detection and fix remaining violations
- Extract exception var names to class constant EXCEPTION_VAR_NAMES
- Add _string_concat_references_exception() to detect "Error: " + str(e)
- Fix 7 remaining files in examples/ and tests/ that failed CI
* fix: restore missing space in notification manager log message
581 lines
20 KiB
Python
Executable File
581 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# This script should be run from the project root directory using:
|
|
# cd /path/to/local-deep-research
|
|
# python -m examples.optimization.strategy_benchmark_plan
|
|
"""
|
|
Strategy Benchmark Plan - Comprehensive Optuna-based optimization for search strategies
|
|
|
|
This benchmark specifically focuses on comparing the iterdrag and source_based strategies
|
|
with 500 examples per experiment to ensure statistically significant results.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import random
|
|
import sys
|
|
import time
|
|
from datetime import datetime, UTC
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Tuple
|
|
|
|
from loguru import logger
|
|
|
|
# Add the src directory to the Python path before local imports
|
|
project_root = str(Path(__file__).parent.parent.parent.resolve())
|
|
sys.path.insert(0, str(Path(project_root) / "src"))
|
|
|
|
# Now we can import from the local project
|
|
from local_deep_research.benchmarks.optimization.optuna_optimizer import ( # noqa: E402
|
|
OptunaOptimizer,
|
|
)
|
|
|
|
# Logger is already imported from loguru at the top
|
|
|
|
# Number of examples to use in each benchmark experiment
|
|
NUM_EXAMPLES = 500
|
|
|
|
|
|
def progress_callback(trial_num, total_trials, data):
|
|
"""Progress callback for optimization"""
|
|
print(f"Progress: {trial_num}/{total_trials} - {data}")
|
|
|
|
|
|
def run_strategy_comparison():
|
|
"""
|
|
Run a comprehensive comparison between iterdrag and source_based strategies.
|
|
Uses a large sample size (500 examples) for statistical significance.
|
|
"""
|
|
# Verify LLM and search database settings before proceeding
|
|
try:
|
|
from local_deep_research.config.llm_config import get_llm
|
|
from local_deep_research.config.search_config import get_search
|
|
from local_deep_research.utilities.db_utils import get_db_setting
|
|
|
|
# Try to initialize LLM and search engine to check configuration
|
|
llm = get_llm()
|
|
search = get_search()
|
|
|
|
# Get relevant DB settings
|
|
try:
|
|
iterations = get_db_setting("search.iterations") or 3
|
|
questions_per_iteration = (
|
|
get_db_setting("search.questions_per_iteration") or 3
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Error getting DB settings: {e}")
|
|
iterations = 3
|
|
questions_per_iteration = 3
|
|
|
|
logger.info("Successfully connected to database")
|
|
logger.info(f"Using LLM: {llm.__class__.__name__}")
|
|
logger.info(f"Using search engine: {search.__class__.__name__}")
|
|
logger.info(f"Default iterations from DB: {iterations}")
|
|
logger.info(
|
|
f"Default questions per iteration from DB: {questions_per_iteration}"
|
|
)
|
|
except Exception as e:
|
|
logger.exception("Error initializing LLM or search settings")
|
|
logger.info("Please check your database configuration")
|
|
return {"error": str(e)}
|
|
timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
|
|
base_output_dir = f"strategy_benchmark_results_{timestamp}"
|
|
os.makedirs(base_output_dir, exist_ok=True)
|
|
|
|
# Define test query
|
|
query = "What are the latest developments in fusion energy research?"
|
|
|
|
# Track execution stats
|
|
execution_stats = {"start_time": time.time(), "experiments": []}
|
|
|
|
# Define parameter space specific to strategy comparison
|
|
strategy_param_space = {
|
|
"search_strategy": {
|
|
"type": "categorical",
|
|
"choices": ["iterdrag", "source_based"],
|
|
},
|
|
"iterations": {
|
|
"type": "int",
|
|
"low": 1,
|
|
"high": 3,
|
|
"step": 1,
|
|
},
|
|
"questions_per_iteration": {
|
|
"type": "int",
|
|
"low": 1,
|
|
"high": 5,
|
|
"step": 1,
|
|
},
|
|
"max_results": {
|
|
"type": "int",
|
|
"low": 10,
|
|
"high": 50,
|
|
"step": 10,
|
|
},
|
|
}
|
|
|
|
# Common settings for all experiments
|
|
common_settings = {
|
|
"query": query,
|
|
"n_trials": 30, # Optuna trials per experiment
|
|
"n_jobs": 1, # Run one job at a time for consistent resource measurement
|
|
"timeout": 3600, # 1 hour timeout per experiment
|
|
"progress_callback": progress_callback,
|
|
}
|
|
|
|
# ====== EXPERIMENT 1: Quality-focused optimization ======
|
|
logger.info("Starting quality-focused benchmark with 500 examples")
|
|
quality_output_dir = str(Path(base_output_dir) / "quality_focused")
|
|
Path(quality_output_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create optimizer for quality
|
|
quality_optimizer = OptunaOptimizer(
|
|
base_query=query,
|
|
output_dir=quality_output_dir,
|
|
n_trials=common_settings["n_trials"],
|
|
timeout=common_settings["timeout"],
|
|
n_jobs=common_settings["n_jobs"],
|
|
progress_callback=common_settings["progress_callback"],
|
|
study_name="strategy_quality_benchmark",
|
|
optimization_metrics=["quality", "speed"],
|
|
metric_weights={"quality": 0.9, "speed": 0.1},
|
|
num_examples=NUM_EXAMPLES, # Use 500 examples for robust evaluation
|
|
)
|
|
|
|
# Run quality optimization
|
|
quality_start = time.time()
|
|
best_quality_params, best_quality_score = quality_optimizer.optimize(
|
|
strategy_param_space
|
|
)
|
|
quality_end = time.time()
|
|
|
|
quality_result = {
|
|
"experiment": "quality_focused",
|
|
"best_params": best_quality_params,
|
|
"best_score": best_quality_score,
|
|
"duration_seconds": quality_end - quality_start,
|
|
}
|
|
execution_stats["experiments"].append(quality_result)
|
|
|
|
# Log and save results
|
|
logger.info(f"Quality benchmark complete: {best_quality_params}")
|
|
logger.info(f"Best quality score: {best_quality_score}")
|
|
logger.info(f"Duration: {quality_end - quality_start} seconds")
|
|
|
|
with open(Path(quality_output_dir) / "results.json", "w") as f:
|
|
json.dump(quality_result, f, indent=2)
|
|
|
|
# ====== EXPERIMENT 2: Speed-focused optimization ======
|
|
logger.info("Starting speed-focused benchmark with 500 examples")
|
|
speed_output_dir = str(Path(base_output_dir) / "speed_focused")
|
|
Path(speed_output_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create optimizer for speed
|
|
speed_optimizer = OptunaOptimizer(
|
|
base_query=query,
|
|
output_dir=speed_output_dir,
|
|
n_trials=common_settings["n_trials"],
|
|
timeout=common_settings["timeout"],
|
|
n_jobs=common_settings["n_jobs"],
|
|
progress_callback=common_settings["progress_callback"],
|
|
study_name="strategy_speed_benchmark",
|
|
optimization_metrics=["quality", "speed"],
|
|
metric_weights={"quality": 0.2, "speed": 0.8},
|
|
num_examples=NUM_EXAMPLES, # Use 500 examples for robust evaluation
|
|
)
|
|
|
|
# Run speed optimization
|
|
speed_start = time.time()
|
|
best_speed_params, best_speed_score = speed_optimizer.optimize(
|
|
strategy_param_space
|
|
)
|
|
speed_end = time.time()
|
|
|
|
speed_result = {
|
|
"experiment": "speed_focused",
|
|
"best_params": best_speed_params,
|
|
"best_score": best_speed_score,
|
|
"duration_seconds": speed_end - speed_start,
|
|
}
|
|
execution_stats["experiments"].append(speed_result)
|
|
|
|
# Log and save results
|
|
logger.info(f"Speed benchmark complete: {best_speed_params}")
|
|
logger.info(f"Best speed score: {best_speed_score}")
|
|
logger.info(f"Duration: {speed_end - speed_start} seconds")
|
|
|
|
with open(Path(speed_output_dir) / "results.json", "w") as f:
|
|
json.dump(speed_result, f, indent=2)
|
|
|
|
# ====== EXPERIMENT 3: Balanced optimization ======
|
|
logger.info("Starting balanced benchmark with 500 examples")
|
|
balanced_output_dir = str(Path(base_output_dir) / "balanced")
|
|
Path(balanced_output_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create optimizer for balanced approach
|
|
balanced_optimizer = OptunaOptimizer(
|
|
base_query=query,
|
|
output_dir=balanced_output_dir,
|
|
n_trials=common_settings["n_trials"],
|
|
timeout=common_settings["timeout"],
|
|
n_jobs=common_settings["n_jobs"],
|
|
progress_callback=common_settings["progress_callback"],
|
|
study_name="strategy_balanced_benchmark",
|
|
optimization_metrics=["quality", "speed", "resource"],
|
|
metric_weights={"quality": 0.4, "speed": 0.3, "resource": 0.3},
|
|
num_examples=NUM_EXAMPLES, # Use 500 examples for robust evaluation
|
|
)
|
|
|
|
# Run balanced optimization
|
|
balanced_start = time.time()
|
|
best_balanced_params, best_balanced_score = balanced_optimizer.optimize(
|
|
strategy_param_space
|
|
)
|
|
balanced_end = time.time()
|
|
|
|
balanced_result = {
|
|
"experiment": "balanced",
|
|
"best_params": best_balanced_params,
|
|
"best_score": best_balanced_score,
|
|
"duration_seconds": balanced_end - balanced_start,
|
|
}
|
|
execution_stats["experiments"].append(balanced_result)
|
|
|
|
# Log and save results
|
|
logger.info(f"Balanced benchmark complete: {best_balanced_params}")
|
|
logger.info(f"Best balanced score: {best_balanced_score}")
|
|
logger.info(f"Duration: {balanced_end - balanced_start} seconds")
|
|
|
|
with open(Path(balanced_output_dir) / "results.json", "w") as f:
|
|
json.dump(balanced_result, f, indent=2)
|
|
|
|
# ====== EXPERIMENT 4: Multi-Benchmark (SimpleQA + BrowseComp) ======
|
|
logger.info("Starting multi-benchmark optimization with 500 examples")
|
|
multi_output_dir = str(Path(base_output_dir) / "multi_benchmark")
|
|
Path(multi_output_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create optimizer with multi-benchmark weights
|
|
multi_optimizer = OptunaOptimizer(
|
|
base_query=query,
|
|
output_dir=multi_output_dir,
|
|
n_trials=common_settings["n_trials"],
|
|
timeout=common_settings["timeout"],
|
|
n_jobs=common_settings["n_jobs"],
|
|
progress_callback=common_settings["progress_callback"],
|
|
study_name="strategy_multi_benchmark",
|
|
optimization_metrics=["quality", "speed"],
|
|
metric_weights={"quality": 0.6, "speed": 0.4},
|
|
benchmark_weights={"simpleqa": 0.6, "browsecomp": 0.4},
|
|
num_examples=NUM_EXAMPLES, # Use 500 examples for robust evaluation
|
|
)
|
|
|
|
# Run multi-benchmark optimization
|
|
multi_start = time.time()
|
|
best_multi_params, best_multi_score = multi_optimizer.optimize(
|
|
strategy_param_space
|
|
)
|
|
multi_end = time.time()
|
|
|
|
multi_result = {
|
|
"experiment": "multi_benchmark",
|
|
"best_params": best_multi_params,
|
|
"best_score": best_multi_score,
|
|
"duration_seconds": multi_end - multi_start,
|
|
}
|
|
execution_stats["experiments"].append(multi_result)
|
|
|
|
# Log and save results
|
|
logger.info(f"Multi-benchmark complete: {best_multi_params}")
|
|
logger.info(f"Best multi-benchmark score: {best_multi_score}")
|
|
logger.info(f"Duration: {multi_end - multi_start} seconds")
|
|
|
|
with open(Path(multi_output_dir) / "results.json", "w") as f:
|
|
json.dump(multi_result, f, indent=2)
|
|
|
|
# ====== Save summary of all executions ======
|
|
execution_stats["total_duration"] = (
|
|
time.time() - execution_stats["start_time"]
|
|
)
|
|
execution_stats["timestamp"] = timestamp
|
|
|
|
with open(Path(base_output_dir) / "summary.json", "w") as f:
|
|
json.dump(execution_stats, f, indent=2)
|
|
|
|
# Generate summary report
|
|
generate_summary_report(base_output_dir, execution_stats)
|
|
|
|
return execution_stats
|
|
|
|
|
|
def generate_summary_report(base_dir, stats):
|
|
"""Generate a human-readable summary report of all benchmarks"""
|
|
summary_text = f"""
|
|
# Strategy Benchmark Results Summary
|
|
|
|
## Overview
|
|
- **Date:** {datetime.fromtimestamp(stats["start_time"]).strftime("%Y-%m-%d %H:%M:%S")}
|
|
- **Total Duration:** {stats["total_duration"] / 3600:.2f} hours
|
|
- **Number of Examples per Experiment:** {NUM_EXAMPLES}
|
|
|
|
## Experiment Results
|
|
|
|
"""
|
|
# Add detailed results for each experiment
|
|
for exp in stats["experiments"]:
|
|
summary_text += f"""### {exp["experiment"].replace("_", " ").title()}
|
|
- **Best Parameters:** {json.dumps(exp["best_params"], indent=2)}
|
|
- **Best Score:** {exp["best_score"]:.4f}
|
|
- **Duration:** {exp["duration_seconds"] / 60:.2f} minutes
|
|
|
|
"""
|
|
|
|
summary_text += """
|
|
## Strategy Comparison
|
|
|
|
| Metric Focus | Best Strategy | Other Parameters | Score |
|
|
|--------------|--------------|------------------|-------|
|
|
"""
|
|
|
|
for exp in stats["experiments"]:
|
|
best_strategy = exp["best_params"].get("search_strategy", "unknown")
|
|
other_params = {
|
|
k: v
|
|
for k, v in exp["best_params"].items()
|
|
if k != "search_strategy"
|
|
}
|
|
summary_text += f"| {exp['experiment'].replace('_', ' ').title()} | {best_strategy} | {other_params} | {exp['best_score']:.4f} |\n"
|
|
|
|
summary_text += """
|
|
## Analysis
|
|
|
|
This benchmark compared the performance of iterdrag and source_based strategies across different optimization goals:
|
|
- Quality-focused: Prioritizes result quality (90%) over speed (10%)
|
|
- Speed-focused: Prioritizes execution speed (80%) over quality (20%)
|
|
- Balanced: Balances quality (40%), speed (30%), and resource usage (30%)
|
|
- Multi-benchmark: Uses weighted combination of SimpleQA (60%) and BrowseComp (40%)
|
|
|
|
The results indicate which strategy is better suited for each optimization goal when using a statistically
|
|
significant sample size of 500 examples per experiment.
|
|
"""
|
|
|
|
# Write summary to file
|
|
with open(Path(base_dir) / "summary_report.md", "w") as f:
|
|
f.write(summary_text)
|
|
|
|
|
|
def run_strategy_simulation(num_examples=10):
|
|
"""
|
|
Run a smaller simulation of the strategy benchmark with fewer examples
|
|
for testing purposes or quick comparisons.
|
|
|
|
This fallback simulation mode doesn't require actual database or LLM access,
|
|
making it useful for testing the script structure.
|
|
"""
|
|
timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
|
|
sim_output_dir = f"strategy_sim_results_{timestamp}"
|
|
os.makedirs(sim_output_dir, exist_ok=True)
|
|
|
|
# Define test query
|
|
query = "What are the latest developments in fusion energy research?"
|
|
|
|
# Define parameter space limited to strategies
|
|
strategy_param_space = {
|
|
"search_strategy": {
|
|
"type": "categorical",
|
|
"choices": ["iterdrag", "source_based"],
|
|
},
|
|
"iterations": {
|
|
"type": "int",
|
|
"low": 1,
|
|
"high": 2,
|
|
"step": 1,
|
|
},
|
|
}
|
|
|
|
try:
|
|
# Try to use real optimizer if available
|
|
logger.info("Attempting to use real optimizer...")
|
|
|
|
# Check if we can access necessary components
|
|
from local_deep_research.config.llm_config import get_llm
|
|
from local_deep_research.config.search_config import get_search
|
|
|
|
# Try to initialize LLM and search engine to check configuration
|
|
llm = get_llm()
|
|
search = get_search()
|
|
|
|
logger.info(
|
|
f"Connected to LLM ({llm.__class__.__name__}) and search ({search.__class__.__name__})"
|
|
)
|
|
|
|
# Create optimizer for simulation
|
|
sim_optimizer = OptunaOptimizer(
|
|
base_query=query,
|
|
output_dir=sim_output_dir,
|
|
n_trials=5, # Just a few trials for simulation
|
|
timeout=600, # 10 minutes timeout
|
|
n_jobs=1,
|
|
study_name="strategy_simulation",
|
|
optimization_metrics=["quality", "speed"],
|
|
metric_weights={"quality": 0.5, "speed": 0.5},
|
|
num_examples=num_examples, # Use fewer examples for simulation
|
|
)
|
|
|
|
# Run simulation
|
|
best_params, best_score = sim_optimizer.optimize(strategy_param_space)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Could not initialize real optimizer: {e!s}")
|
|
logger.warning(
|
|
"Falling back to pure simulation mode (no real benchmarks)"
|
|
)
|
|
|
|
# Simulate optimization if real system is unavailable
|
|
logger.info(
|
|
"Running purely simulated optimization (no real benchmarks)"
|
|
)
|
|
best_params, best_score = simulate_optimization(
|
|
strategy_param_space,
|
|
n_trials=5,
|
|
metric_weights={"quality": 0.5, "speed": 0.5},
|
|
)
|
|
|
|
# Log and save results
|
|
logger.info(f"Simulation complete: {best_params}")
|
|
logger.info(f"Best simulation score: {best_score}")
|
|
|
|
sim_result = {
|
|
"best_params": best_params,
|
|
"best_score": best_score,
|
|
}
|
|
|
|
with open(Path(sim_output_dir) / "simulation_results.json", "w") as f:
|
|
json.dump(sim_result, f, indent=2)
|
|
|
|
return sim_result
|
|
|
|
|
|
def simulate_optimization(
|
|
param_space: Dict[str, Any],
|
|
n_trials: int = 5,
|
|
metric_weights: Dict[str, float] = None,
|
|
) -> Tuple[Dict[str, Any], float]:
|
|
"""
|
|
Simulate an optimization process without actually running benchmarks.
|
|
This is just for demonstration/testing purposes when the real system is unavailable.
|
|
|
|
Args:
|
|
param_space: Dictionary defining parameter search spaces
|
|
n_trials: Number of simulated trials
|
|
metric_weights: Weights for quality vs speed metrics
|
|
|
|
Returns:
|
|
Tuple of (best_parameters, best_score)
|
|
"""
|
|
if metric_weights is None:
|
|
metric_weights = {"quality": 0.5, "speed": 0.5}
|
|
|
|
logger.info(f"Starting simulated optimization with {n_trials} trials")
|
|
logger.info(f"Parameter space: {param_space}")
|
|
logger.info(f"Metric weights: {metric_weights}")
|
|
|
|
# Generate random trials
|
|
best_score = 0.0
|
|
best_params = {}
|
|
|
|
for i in range(n_trials):
|
|
# Generate random parameters
|
|
params = {}
|
|
for param_name, param_config in param_space.items():
|
|
if param_config.get("type") == "int":
|
|
params[param_name] = random.randint(
|
|
param_config.get("low", 1), param_config.get("high", 5)
|
|
)
|
|
elif param_config.get("type") == "categorical":
|
|
params[param_name] = random.choice(
|
|
param_config.get("choices", ["standard"])
|
|
)
|
|
|
|
logger.info(f"Trial {i + 1}: Testing parameters: {params}")
|
|
|
|
# Simulate execution delay
|
|
time.sleep(0.5)
|
|
|
|
# Simulate metrics for different strategies
|
|
quality_score = 0.0
|
|
speed_score = 0.0
|
|
|
|
# Generate strategy-specific simulated scores
|
|
if params.get("search_strategy") == "iterdrag":
|
|
# IterDRAG typically has higher quality but lower speed
|
|
quality_score = random.uniform(0.7, 0.95)
|
|
speed_score = random.uniform(0.4, 0.7)
|
|
elif params.get("search_strategy") == "source_based":
|
|
# Source-based typically has medium quality but higher speed
|
|
quality_score = random.uniform(0.6, 0.85)
|
|
speed_score = random.uniform(0.6, 0.9)
|
|
else:
|
|
# Other strategies
|
|
quality_score = random.uniform(0.5, 0.9)
|
|
speed_score = random.uniform(0.5, 0.9)
|
|
|
|
# More iterations generally means higher quality but lower speed
|
|
iterations = params.get("iterations", 1)
|
|
quality_score += (
|
|
iterations * 0.05
|
|
) # More iterations slightly improves quality
|
|
speed_score -= (
|
|
iterations * 0.15
|
|
) # More iterations significantly reduces speed
|
|
|
|
# Normalize scores to 0-1 range
|
|
quality_score = max(0.0, min(1.0, quality_score))
|
|
speed_score = max(0.0, min(1.0, speed_score))
|
|
|
|
# Calculate weighted score based on metric weights
|
|
combined_score = quality_score * metric_weights.get(
|
|
"quality", 0.5
|
|
) + speed_score * metric_weights.get("speed", 0.5)
|
|
|
|
logger.info(
|
|
f"Trial {i + 1}: Quality: {quality_score:.2f}, Speed: {speed_score:.2f}, Score: {combined_score:.2f}"
|
|
)
|
|
|
|
# Update best parameters if this trial is better
|
|
if combined_score > best_score:
|
|
best_score = combined_score
|
|
best_params = params.copy()
|
|
logger.info(
|
|
f"New best parameters found: {best_params} with score: {best_score:.2f}"
|
|
)
|
|
|
|
return best_params, best_score
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Run strategy benchmarks")
|
|
parser.add_argument(
|
|
"--simulate",
|
|
action="store_true",
|
|
help="Run a quick simulation instead of full benchmark",
|
|
)
|
|
parser.add_argument(
|
|
"--examples",
|
|
type=int,
|
|
default=NUM_EXAMPLES,
|
|
help=f"Number of examples to use (default: {NUM_EXAMPLES})",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.simulate:
|
|
logger.info(f"Running simulation with {args.examples} examples")
|
|
run_strategy_simulation(args.examples)
|
|
else:
|
|
logger.info(f"Running full benchmark with {args.examples} examples")
|
|
NUM_EXAMPLES = args.examples # Override global constant
|
|
|
|
# Just run the benchmark function directly
|
|
run_strategy_comparison()
|