mirror of
https://github.com/LearningCircuit/local-deep-research.git
synced 2026-06-16 03:51:07 +03:00
65b2383eb7
* Revert "Revert "fix: add SQLCipher 4.x compatibility for cipher pragma orderi…"
This reverts commit d72b7ae668.
* ci: add backwards compatibility workflow for SQLCipher encryption
Add a CI workflow that verifies database encryption backwards compatibility:
- Runs encryption constants tests on PRs touching database files
- Runs full PyPI version compatibility test on main/releases
- Triggers on schedule (weekly) to catch dependency drift
- Path-filtered to only run when relevant files change
This prevents regressions like salt or KDF parameter changes that would
break existing encrypted databases.
* ci: add backwards compatibility to security release gate
- Add workflow_call trigger to backwards-compatibility.yml
- Include backwards-compatibility check in security-release-gate.yml
- Run full PyPI compatibility test on releases (not just encryption constants)
This ensures database encryption changes can't break existing user databases
in a release.
* test: increase timeouts for PyPI backwards compatibility test
- Add pytest.mark.timeout(600) for the test function
- Increase pip install timeout from 300s to 600s
The package has many dependencies (numpy, torch, etc.) which take
significant time to install in a fresh venv.
* ci: fix shellcheck SC2129 in backwards-compatibility workflow
Group consecutive GITHUB_OUTPUT redirects into a single block
to satisfy actionlint/shellcheck SC2129 style check.
* fix: comprehensive SQLCipher security and correctness improvements
- Fix critical PRAGMA ordering: cipher_default_* before key for new DBs,
cipher_* after key for existing DBs (per Zetetic SQLCipher 4.x docs)
- Replace unbounded @cache with @lru_cache(maxsize=8) to prevent memory leaks
- Add thread safety: RLock for connections dict, Lock for cipher_default globals
- Pre-derive hex keys before closures to avoid capturing plaintext passwords
- Add connection cleanup on failure (close conn, remove partial DB files)
- Add engine.dispose() on failed open_user_database()
- Whitespace password validation (reject " " as password)
- Remove cipher_memory_security=OFF (SQLCipher >=4.5 defaults to OFF)
- CI-aware KDF minimum: 100k production, relaxed in test environments
- Add set_sqlcipher_key_from_hex(), get_sqlcipher_version() utilities
- Replace direct db_manager.connections dict access with is_user_connected()
- Update harden-runner to v2.14.1, improve CI error handling
- Fix all except Exception: pass patterns to re-raise AssertionError in tests
- Update all test files for renamed functions and correct PRAGMA ordering
* fix: address race condition in get_session() and exception swallowing in test
Move sessionmaker + session creation inside _connections_lock to prevent
race with close_user_database() disposing the engine. Also fix exception
handler in test_sqlcipher_integration.py to re-raise AssertionError.
* fix: address review issues in SQLCipher 4.x compatibility PR
- Update 3 broken web test files (test_queue_manager, test_decorators,
test_session_cleanup) to mock is_user_connected()/get_session() instead
of the removed connections.get() API
- Wire backwards-compatibility.yml into security-release-gate.yml so
encryption compat checks actually run during releases
- Add missing apply_sqlcipher_pragmas() call in _check_encryption_available()
to properly set kdf_iter after key on the test connection
- Replace generic CI/TESTING env var checks with PYTEST_CURRENT_TEST and
LDR_TEST_MODE to prevent accidental KDF weakening in production
- Add timeout-minutes to both backwards-compatibility CI jobs
436 lines
16 KiB
Python
436 lines
16 KiB
Python
"""
|
|
Tests for SQLCipher missing scenario.
|
|
|
|
Verifies that users get helpful error messages when SQLCipher is not installed
|
|
and that the LDR_BOOTSTRAP_ALLOW_UNENCRYPTED workaround works.
|
|
"""
|
|
|
|
import os
|
|
import pytest
|
|
from unittest.mock import patch
|
|
|
|
from tests.test_utils import add_src_to_path
|
|
|
|
add_src_to_path()
|
|
|
|
|
|
def _clear_allow_unencrypted_env():
|
|
"""Clear both canonical and deprecated env vars."""
|
|
os.environ.pop("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED", None)
|
|
os.environ.pop("LDR_ALLOW_UNENCRYPTED", None)
|
|
|
|
|
|
class TestSQLCipherMissing:
|
|
"""Test behavior when SQLCipher is not available."""
|
|
|
|
def test_error_message_mentions_sqlcipher(self):
|
|
"""Error message should mention SQLCipher so users know what's missing."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
old_canonical = os.environ.pop("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED", None)
|
|
old_deprecated = os.environ.pop("LDR_ALLOW_UNENCRYPTED", None)
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
with pytest.raises(RuntimeError) as exc_info:
|
|
DatabaseManager()
|
|
|
|
error_msg = str(exc_info.value)
|
|
assert "SQLCipher" in error_msg, (
|
|
f"Error should mention SQLCipher. Got: {error_msg}"
|
|
)
|
|
finally:
|
|
if old_canonical is not None:
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = old_canonical
|
|
if old_deprecated is not None:
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = old_deprecated
|
|
|
|
def test_error_message_mentions_workaround(self):
|
|
"""Error message should mention LDR_BOOTSTRAP_ALLOW_UNENCRYPTED workaround."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
old_canonical = os.environ.pop("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED", None)
|
|
old_deprecated = os.environ.pop("LDR_ALLOW_UNENCRYPTED", None)
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
with pytest.raises(RuntimeError) as exc_info:
|
|
DatabaseManager()
|
|
|
|
error_msg = str(exc_info.value)
|
|
assert "LDR_BOOTSTRAP_ALLOW_UNENCRYPTED" in error_msg, (
|
|
f"Error should mention workaround. Got: {error_msg}"
|
|
)
|
|
finally:
|
|
if old_canonical is not None:
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = old_canonical
|
|
if old_deprecated is not None:
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = old_deprecated
|
|
|
|
def test_canonical_workaround_allows_startup_without_encryption(self):
|
|
"""LDR_BOOTSTRAP_ALLOW_UNENCRYPTED=true should allow startup without SQLCipher."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
old_canonical = os.environ.get("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED")
|
|
old_deprecated = os.environ.get("LDR_ALLOW_UNENCRYPTED")
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = "true"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False, (
|
|
"With workaround and no SQLCipher, has_encryption should be False"
|
|
)
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
if old_canonical is not None:
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = old_canonical
|
|
if old_deprecated is not None:
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = old_deprecated
|
|
|
|
def test_deprecated_workaround_still_works(self):
|
|
"""Deprecated LDR_ALLOW_UNENCRYPTED=true should still allow startup (backward compat)."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
old_canonical = os.environ.get("LDR_BOOTSTRAP_ALLOW_UNENCRYPTED")
|
|
old_deprecated = os.environ.get("LDR_ALLOW_UNENCRYPTED")
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise (backward compatibility)
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False, (
|
|
"With deprecated workaround and no SQLCipher, has_encryption should be False"
|
|
)
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
if old_canonical is not None:
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = old_canonical
|
|
if old_deprecated is not None:
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = old_deprecated
|
|
|
|
def test_db_manager_has_encryption_is_boolean(self):
|
|
"""db_manager.has_encryption should be a boolean."""
|
|
from local_deep_research.database.encrypted_db import db_manager
|
|
|
|
assert isinstance(db_manager.has_encryption, bool), (
|
|
f"has_encryption should be bool, got {type(db_manager.has_encryption)}"
|
|
)
|
|
|
|
|
|
class TestAllowUnencryptedEdgeCases:
|
|
"""Edge cases for LDR_BOOTSTRAP_ALLOW_UNENCRYPTED fallback.
|
|
|
|
The allow_unencrypted setting uses the BooleanSetting from the env
|
|
settings registry, which accepts "true", "1", "yes", "on", "enabled"
|
|
(case-insensitive) as truthy values and handles deprecated alias
|
|
fallback automatically.
|
|
"""
|
|
|
|
def test_canonical_empty_string_does_not_fall_through(self):
|
|
"""Empty string canonical should NOT fall through to deprecated.
|
|
|
|
When LDR_BOOTSTRAP_ALLOW_UNENCRYPTED="" is set, it takes precedence
|
|
and the check (empty_string or "").lower() == "true" is False.
|
|
"""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = ""
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should raise because empty string is not "true"
|
|
with pytest.raises(RuntimeError):
|
|
DatabaseManager()
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_uppercase_true_not_accepted(self):
|
|
"""'TRUE' (uppercase) is not accepted as true.
|
|
|
|
The code uses .lower() == "true", so "TRUE".lower() == "true" is True.
|
|
"""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = "TRUE"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise - "TRUE".lower() == "true" is True
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_mixed_case_true_accepted(self):
|
|
"""'True' (mixed case) is accepted as true due to .lower()."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = "True"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise - "True".lower() == "true" is True
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_whitespace_padded_true_not_accepted(self):
|
|
"""' true ' (whitespace padded) is not accepted as true.
|
|
|
|
The code doesn't strip whitespace, so " true " != "true".
|
|
"""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = " true "
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should raise - " true " != "true"
|
|
with pytest.raises(RuntimeError):
|
|
DatabaseManager()
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_yes_accepted_as_true(self):
|
|
"""'yes' is accepted as true by BooleanSetting."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = "yes"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise - BooleanSetting accepts "yes"
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_one_accepted_as_true(self):
|
|
"""'1' is accepted as true by BooleanSetting."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = "1"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise - BooleanSetting accepts "1"
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_canonical_false_does_not_check_deprecated(self):
|
|
"""canonical='false' should NOT fall through to deprecated='true'.
|
|
|
|
Once canonical is set (to any value), deprecated is not checked.
|
|
"""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = "false"
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should raise - canonical "false" takes precedence
|
|
with pytest.raises(RuntimeError):
|
|
DatabaseManager()
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_empty_canonical_with_deprecated_true_raises(self):
|
|
"""Empty canonical with deprecated='true' - empty takes precedence.
|
|
|
|
When canonical is "", the deprecated is not checked because
|
|
canonical is "set" (to empty string).
|
|
"""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_BOOTSTRAP_ALLOW_UNENCRYPTED"] = ""
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should raise - empty string canonical takes precedence
|
|
with pytest.raises(RuntimeError):
|
|
DatabaseManager()
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_only_deprecated_set_to_true_works(self):
|
|
"""When only deprecated is set to 'true', it should work."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = "true"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise - deprecated "true" works
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_deprecated_uppercase_true_accepted(self):
|
|
"""Deprecated 'TRUE' (uppercase) works due to .lower()."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
os.environ["LDR_ALLOW_UNENCRYPTED"] = "TRUE"
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
# Should NOT raise - "TRUE".lower() == "true"
|
|
manager = DatabaseManager()
|
|
assert manager.has_encryption is False
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|
|
|
|
def test_neither_set_raises_runtime_error(self):
|
|
"""When neither canonical nor deprecated is set, RuntimeError is raised."""
|
|
from local_deep_research.database.encrypted_db import (
|
|
DatabaseManager,
|
|
)
|
|
|
|
_clear_allow_unencrypted_env()
|
|
|
|
try:
|
|
with patch(
|
|
"local_deep_research.database.encrypted_db.get_sqlcipher_module"
|
|
) as mock_get:
|
|
mock_get.side_effect = ImportError(
|
|
"No module named 'sqlcipher3'"
|
|
)
|
|
|
|
with pytest.raises(RuntimeError):
|
|
DatabaseManager()
|
|
finally:
|
|
_clear_allow_unencrypted_env()
|