from __future__ import annotations import fcntl import json import os import sqlite3 from datetime import UTC, datetime, timedelta from pathlib import Path import pytest from codexpi.storage_reconciliation import ( DestructiveValidationRequired, StorageReconciler, ) def _database(path: Path, *, worktree: Path | None = None, active: bool = False) -> None: connection = sqlite3.connect(path) connection.executescript( """ CREATE TABLE conversations ( id TEXT, worktree_path TEXT, event_file TEXT, codex_thread_id TEXT, status TEXT, worker_pid INTEGER, updated_at TEXT ); CREATE TABLE prompt_queue (conversation_id TEXT, status TEXT); """ ) if worktree: connection.execute( "INSERT INTO conversations VALUES(?,?,?,?,?,?,?)", ("conversation", str(worktree), None, "thread-live", "running" if active else "idle", os.getpid() if active else None, datetime.now(UTC).isoformat()), ) if active: connection.execute("INSERT INTO prompt_queue VALUES(?,?)", ("conversation", "running")) connection.commit() connection.close() def _instance(root: Path) -> Path: (root / "data" / "worktrees").mkdir(parents=True) (root / "data" / "sessions").mkdir() (root / "data" / "backups").mkdir() return root def test_inventory_excludes_live_and_active_references_and_estimates_duplicates(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") live = primary / "data" / "worktrees" / "live" live.mkdir() (live / "result.txt").write_text("preserve", encoding="utf-8") duplicate_primary = primary / "data" / "backups" / "old" duplicate_secondary = secondary / "data" / "backups" / "old" duplicate_primary.mkdir() duplicate_secondary.mkdir() (duplicate_primary / "archive.txt").write_text("same", encoding="utf-8") (duplicate_secondary / "archive.txt").write_text("same", encoding="utf-8") _database(primary / "data" / "codexpi.db", worktree=live, active=True) _database(secondary / "data" / "codexpi.db") report = StorageReconciler([primary, secondary], now=datetime.now(UTC)).inventory() live_item = next(item for item in report.items if item.path == str(live)) duplicate = next(item for item in report.items if item.path == str(duplicate_secondary)) assert live_item.active is True assert "database_reference" in live_item.exclusions assert "active_session" in live_item.exclusions assert live_item.reclaimable is False assert duplicate.duplicate_of is not None assert duplicate.reclaimable is True assert report.reclaimable_bytes == duplicate.size_bytes def test_inventory_protects_candidate_when_active_reference_is_its_parent(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): candidate = root / "data" / "worktrees" / "child" candidate.mkdir() (candidate / "same.txt").write_text("same", encoding="utf-8") _database(primary / "data" / "codexpi.db") _database( secondary / "data" / "codexpi.db", worktree=secondary / "data" / "worktrees", active=True, ) report = StorageReconciler([primary, secondary]).inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") assert candidate.active is True assert "database_reference" in candidate.exclusions assert "active_session" in candidate.exclusions assert candidate.reclaimable is False def test_inventory_protects_instance_when_database_is_missing(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") _database(primary / "data" / "codexpi.db") for root in (primary, secondary): candidate = root / "data" / "backups" / "same" candidate.mkdir() (candidate / "same.txt").write_text("same", encoding="utf-8") report = StorageReconciler([primary, secondary]).inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") assert "database_unavailable" in candidate.exclusions assert candidate.reclaimable is False def test_inventory_protects_instance_when_database_schema_is_empty(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") _database(primary / "data" / "codexpi.db") sqlite3.connect(secondary / "data" / "codexpi.db").close() for root in (primary, secondary): candidate = root / "data" / "backups" / "same" candidate.mkdir() (candidate / "same.txt").write_text("same", encoding="utf-8") report = StorageReconciler([primary, secondary]).inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") assert "database_unavailable" in candidate.exclusions assert candidate.reclaimable is False def test_inventory_never_traverses_symbolic_data_parent(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") _database(primary / "data" / "codexpi.db") secondary = tmp_path / "CodexPi2" secondary.mkdir() outside_data = tmp_path / "outside-data" _instance(tmp_path / "outside-holder") # Reuse only the generated data layout, then place it outside CodexPi2. generated_data = tmp_path / "outside-holder" / "data" generated_data.rename(outside_data) (tmp_path / "outside-holder").rmdir() _database(outside_data / "codexpi.db") candidate = outside_data / "backups" / "same" candidate.mkdir() (candidate / "same.txt").write_text("must-not-be-read", encoding="utf-8") (secondary / "data").symlink_to(outside_data, target_is_directory=True) report = StorageReconciler([primary, secondary]).inventory() assert all(item.instance != "CodexPi2" for item in report.items) assert str(candidate) not in {item.path for item in report.items} def test_inventory_does_not_follow_symbolic_links(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") outside = tmp_path / "outside" outside.mkdir() (outside / "private.txt").write_text("do not traverse", encoding="utf-8") link = secondary / "data" / "backups" / "linked" link.symlink_to(outside, target_is_directory=True) _database(primary / "data" / "codexpi.db") _database(secondary / "data" / "codexpi.db") report = StorageReconciler([primary, secondary]).inventory() item = next(item for item in report.items if item.path == str(link)) assert item.is_symlink is True assert item.size_bytes == link.lstat().st_size assert item.reclaimable is False assert "symbolic_link" in item.exclusions def test_quarantine_requires_distinct_validation_and_can_be_restored(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") first = primary / "data" / "sessions" / "old.jsonl" second = secondary / "data" / "sessions" / "old.jsonl" first.write_text("identical\n", encoding="utf-8") second.write_text("identical\n", encoding="utf-8") reconciler = StorageReconciler([primary, secondary], grace_days=14) report = reconciler.inventory() candidate = next(item for item in report.items if item.path == str(second)) with pytest.raises(DestructiveValidationRequired): reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=None) receipt = reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) assert not second.exists() assert Path(receipt.quarantine_path).exists() manifest = json.loads((Path(receipt.quarantine_path).parent / "manifest.json").read_text(encoding="utf-8")) assert manifest["original_path"] == str(second) assert datetime.fromisoformat(manifest["purge_eligible_at"]) >= datetime.now(UTC) + timedelta(days=13) with pytest.raises(DestructiveValidationRequired): reconciler.restore(receipt.receipt_id, confirmation=None) reconciler.restore(receipt.receipt_id, confirmation=f"RESTORE:{receipt.receipt_id}") assert second.read_text(encoding="utf-8") == "identical\n" assert not Path(receipt.quarantine_path).exists() def test_quarantine_rechecks_new_active_reference_after_dry_run(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") candidate = root / "data" / "backups" / "same" candidate.mkdir() (candidate / "same.txt").write_text("same", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") connection = sqlite3.connect(secondary / "data" / "codexpi.db") connection.execute( "INSERT INTO conversations VALUES(?,?,?,?,?,?,?)", ("new-active", candidate.path, None, None, "running", os.getpid(), datetime.now(UTC).isoformat()), ) connection.commit() connection.close() with pytest.raises(RuntimeError, match="active|reference|dry-run"): reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) assert Path(candidate.path).exists() def test_quarantine_blocks_concurrent_sqlite_admission_before_move( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") candidate = root / "data" / "backups" / "same" candidate.mkdir() (candidate / "same.txt").write_text("same", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") real_replace = os.replace admitted: list[bool] = [] def racing_replace(source: str | Path, destination: str | Path) -> None: connection = sqlite3.connect(secondary / "data" / "codexpi.db", timeout=0) try: connection.execute( "INSERT INTO conversations VALUES(?,?,?,?,?,?,?)", ("racing", candidate.path, None, None, "running", os.getpid(), datetime.now(UTC).isoformat()), ) connection.commit() admitted.append(True) except sqlite3.OperationalError: admitted.append(False) finally: connection.close() real_replace(source, destination) monkeypatch.setattr(os, "replace", racing_replace) reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) assert admitted == [False] connection = sqlite3.connect(secondary / "data" / "codexpi.db") assert connection.execute("SELECT COUNT(*) FROM conversations WHERE id='racing'").fetchone()[0] == 0 connection.close() def test_quarantine_rejects_instance_root_symbolic_link(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") candidate = root / "data" / "backups" / "same" candidate.mkdir() (candidate / "same.txt").write_text("same", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") real_secondary = tmp_path / "CodexPi2-real" secondary.rename(real_secondary) secondary.symlink_to(real_secondary, target_is_directory=True) with pytest.raises(ValueError, match="symbolic"): reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) assert Path(candidate.path).exists() def test_concurrent_reconciliation_lock_blocks_mutation(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") for root in (primary, secondary): value = root / "data" / "backups" / "duplicate" value.mkdir() (value / "data").write_text("same", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") lock_path = primary / "data" / "storage-reconciliation.lock" lock_path.touch(mode=0o600) with lock_path.open("r+") as held: fcntl.flock(held.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) with pytest.raises(BlockingIOError): reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) def test_failed_audit_rolls_quarantine_back(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") value = root / "data" / "backups" / "duplicate" value.mkdir() (value / "data").write_text("same", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") original = Path(candidate.path) monkeypatch.setattr(reconciler, "_append_audit", lambda _record: (_ for _ in ()).throw(OSError("disk full"))) with pytest.raises(OSError, match="disk full"): reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) assert original.exists() assert not any((primary / "data" / "quarantine").glob("*/payload")) def test_restore_rejects_parent_replaced_by_symbolic_link(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") value = root / "data" / "sessions" / "old.jsonl" value.write_text("same\n", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") receipt = reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) sessions = secondary / "data" / "sessions" sessions.rmdir() outside = tmp_path / "outside" outside.mkdir() sessions.symlink_to(outside, target_is_directory=True) with pytest.raises(ValueError, match="symbolic"): reconciler.restore(receipt.receipt_id, confirmation=f"RESTORE:{receipt.receipt_id}") assert not (outside / "old.jsonl").exists() assert Path(receipt.quarantine_path).exists() def test_restore_rejects_instance_root_replaced_by_symbolic_link(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root in (primary, secondary): _database(root / "data" / "codexpi.db") value = root / "data" / "sessions" / "old.jsonl" value.write_text("same\n", encoding="utf-8") reconciler = StorageReconciler([primary, secondary]) report = reconciler.inventory() candidate = next(item for item in report.items if item.instance == "CodexPi2") receipt = reconciler.quarantine(candidate.item_id, report.plan_id, confirmation=report.destructive_confirmation) real_secondary = tmp_path / "CodexPi2-real" secondary.rename(real_secondary) secondary.symlink_to(real_secondary, target_is_directory=True) with pytest.raises(ValueError, match="symbolic"): reconciler.restore(receipt.receipt_id, confirmation=f"RESTORE:{receipt.receipt_id}") assert not (real_secondary / "data" / "sessions" / "old.jsonl").exists() assert Path(receipt.quarantine_path).exists() def test_browser_cache_version_is_reported_and_only_matching_versions_are_duplicates(tmp_path: Path) -> None: primary = _instance(tmp_path / "CodexPi") secondary = _instance(tmp_path / "CodexPi2") for root, version in ((primary, "132.0.1"), (secondary, "132.0.9")): _database(root / "data" / "codexpi.db") cache = root / "data" / "browser-cache" / "132" / "Cache" cache.mkdir(parents=True) (cache.parent / "chromium.version").write_text(version, encoding="ascii") (cache / "entry").write_text("same", encoding="utf-8") report = StorageReconciler([primary, secondary]).inventory() browser_items = [item for item in report.items if item.kind == "browser_cache"] assert len(browser_items) == 2 assert {item.browser_version for item in browser_items} == {"132"} assert sum(item.reclaimable for item in browser_items) == 1