fix(stabilization): harden attachment lifecycle and agent guard signals (#5420)

* fix: harden stabilization attachment and agent guards

* fix(uploads): preserve durable references during cleanup

* fix(uploads): close cleanup and compaction races

tests/test_agent_rounds_exhausted.py

@@ -68,3 +68,24 @@ def test_no_rounds_exhausted_on_normal_finish(monkeypatch):
     # A plain answer (no tool block) -> done-break on round 1 -> no event.
     events = _run_loop(monkeypatch, "All done, here is your answer.", max_rounds=2)
     assert not any(e.get("type") == "rounds_exhausted" for e in events), events
+
+
+def test_emits_intent_nudge_exhausted_when_cap_is_exhausted(monkeypatch):
+    _patch_common(monkeypatch)
+
+    events = _run_loop(monkeypatch, "Let me check the logs", max_rounds=5)
+
+    guard = next((e for e in events if e.get("type") == "intent_nudge_exhausted"), None)
+    assert guard is not None, events
+    assert guard["reason"] == "intent_without_action_nudge_cap"
+    assert guard["nudges"] == 2
+
+
+def test_emits_loop_breaker_triggered_when_loop_breaker_trips(monkeypatch):
+    _patch_common(monkeypatch)
+
+    events = _run_loop(monkeypatch, "```bash\necho hi\n```", max_rounds=6)
+
+    guard = next((e for e in events if e.get("type") == "loop_breaker_triggered"), None)
+    assert guard is not None, events
+    assert guard["reason"] == "loop_breaker_stall"

tests/test_attachment_refs.py

@@ -0,0 +1,75 @@
+import json
+
+from src.attachment_refs import (
+    attachment_ref,
+    persistable_message_content,
+    search_index_text,
+)
+
+
+def test_persistable_message_content_replaces_inline_media_with_attachment_ref():
+    metadata = {
+        "attachments": [
+            {
+                "id": "abc123.png",
+                "name": "diagram.png",
+                "mime": "image/png",
+                "size": 42,
+                "checksum_sha256": "sha256-digest",
+                "created_at": "2026-07-09T12:00:00",
+                "vision": "A small architecture diagram.",
+            }
+        ]
+    }
+    content = [
+        {"type": "text", "text": "Please inspect this."},
+        {
+            "type": "image_url",
+            "image_url": {"url": "data:image/png;base64," + ("A" * 5000)},
+        },
+    ]
+
+    stored = persistable_message_content(content, metadata)
+
+    assert "base64" not in stored
+    assert "A" * 100 not in stored
+    assert "Please inspect this." in stored
+    assert "Attachment: diagram.png" in stored
+    assert "id=abc123.png" in stored
+    assert "sha256=sha256-digest" in stored
+    assert "A small architecture diagram." in stored
+
+
+def test_search_index_text_strips_legacy_serialized_data_url_blocks():
+    legacy = json.dumps([
+        {"type": "text", "text": "Find this useful caption"},
+        {
+            "type": "image_url",
+            "image_url": {"url": "data:image/jpeg;base64," + ("B" * 4096)},
+        },
+    ])
+
+    indexed = search_index_text(legacy)
+
+    assert indexed == "Find this useful caption\n[1 inline media payload omitted]"
+
+
+def test_attachment_ref_normalizes_hash_aliases():
+    ref = attachment_ref({
+        "id": "file-id",
+        "original_name": "report.pdf",
+        "mime": "application/pdf",
+        "size": 99,
+        "hash": "abc",
+        "uploaded_at": "2026-07-09T12:00:00",
+    })
+
+    assert ref == {
+        "type": "attachment_ref",
+        "attachment_id": "file-id",
+        "name": "report.pdf",
+        "mime": "application/pdf",
+        "size": 99,
+        "checksum_sha256": "abc",
+        "created_at": "2026-07-09T12:00:00",
+    }

tests/test_chat_helpers.py

@@ -233,6 +233,10 @@ def test_build_uploaded_file_manifest_filters_and_nulls_unreadable_paths(monkeyp
         )
 
         assert [item["id"] for item in manifest] == ["good", "outside", "missing"]
+        assert manifest[0]["type"] == "attachment_ref"
+        assert manifest[0]["attachment_id"] == "good"
+        assert manifest[0]["uri"] == "odysseus://attachment/good"
+        assert manifest[0]["read_policy"] == "owner_checked_upload"
         assert os.path.realpath(manifest[0]["path"]) == os.path.realpath(good)
         assert manifest[1]["path"] is None
         assert manifest[2]["path"] is None

tests/test_parse_msg_content_jsonlike_string.py

@@ -1,13 +1,9 @@
-"""A plain text message that merely *looks* like a JSON array of objects must
-NOT be silently re-parsed into a list on reload.
+"""Persistence contracts for JSON-like text and multimodal chat content.
 
-_parse_msg_content de-serializes multimodal (image/audio) content back into a
-list of content blocks. The old heuristic accepted ANY string that started
-with "[{" and contained the substring '"type"'. A user who pasted an API
-schema / sample such as `[{"type": "object", "name": "foo"}]` therefore had
-their text message permanently corrupted into a Python list on the next
-session hydration. The fix restricts the round-trip to lists whose elements
-are all recognized content-block types (text/image_url/audio/...).
+Plain text that resembles a JSON content-block list must remain an exact
+string. Real provider multimodal blocks follow the durable attachment
+contract: readable text plus stable attachment metadata is persisted, while
+raw inline media bytes are omitted.
 """
 import tempfile
 import uuid
@@ -65,16 +61,48 @@ def test_jsonlike_user_string_not_corrupted(manager):
     assert reloaded.history[0].content == text
 
 
-def test_real_multimodal_content_still_round_trips(manager):
+def test_real_multimodal_content_persists_reference_without_base64(manager):
     sid = "sess-" + uuid.uuid4().hex[:8]
     _make_session(sid)
+    attachment_id = "a" * 32 + ".png"
     multimodal = [
         {"type": "text", "text": "what is this?"},
         {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
     ]
-    msgs = [ChatMessage(role="user", content=multimodal)]
+    metadata = {
+        "attachments": [
+            {
+                "id": attachment_id,
+                "name": "diagram.png",
+                "mime": "image/png",
+                "size": 4,
+                "checksum_sha256": "sha256-digest",
+            }
+        ]
+    }
+    msgs = [ChatMessage(role="user", content=multimodal, metadata=metadata)]
     assert manager.replace_messages(sid, msgs) is True
 
+    expected = (
+        "what is this?\n"
+        "[1 inline media payload omitted]\n"
+        f"[Attachment: diagram.png | id={attachment_id} | mime=image/png | "
+        "size=4 bytes | sha256=sha256-digest]"
+    )
+
+    db = _TS()
+    try:
+        stored = db.query(cdb.ChatMessage).filter_by(session_id=sid).one()
+        assert stored.content == expected
+        assert "what is this?" in stored.content
+        assert attachment_id in stored.content
+        assert "data:image/png;base64,AAAA" not in stored.content
+        assert "base64" not in stored.content
+        assert "AAAA" not in stored.content
+    finally:
+        db.close()
+
     manager.sessions.clear()
     reloaded = manager.get_session(sid)
-    assert reloaded.history[0].content == multimodal
+    assert reloaded.history[0].content == expected
+    assert reloaded.history[0].metadata["attachments"][0]["id"] == attachment_id

tests/test_replace_messages_multimodal.py

@@ -1,14 +1,9 @@
-"""replace_messages must JSON-serialize multimodal (list) content.
+"""replace_messages must persist readable, path-free multimodal history.
 
-A chat with an image/audio attachment carries list content. When such a
-chat is compacted, the manual-compaction path calls replace_messages with
-the retained messages. replace_messages wrote message.content straight into
-the Text column, so SQLAlchemy bound the list\'s single-quoted repr. On
-reload _parse_msg_content only de-serializes a string that contains the
-double-quoted "type", so the repr failed the check and the message came
-back as a corrupted string blob - the attachment was destroyed. The
-sibling _persist_message json.dumps-es list content; replace_messages did
-not.
+Live model input may contain provider-specific media blocks and inline data
+URLs. Compaction uses replace_messages for the retained transcript, which must
+store readable text plus stable structured attachment references without
+copying raw base64 payloads into ChatMessage.content.
 """
 import uuid
 
@@ -27,6 +22,7 @@ def manager(monkeypatch):
     monkeypatch.setattr(sm, "SessionLocal", _TS)
     mgr = sm.SessionManager.__new__(sm.SessionManager)
     mgr.sessions = {}
+    mgr.upload_handler = None
     return mgr
 
 
@@ -41,33 +37,71 @@ def _make_session(sid, owner="alice"):
         db.close()
 
 
-def test_multimodal_content_round_trips_through_replace_messages(manager):
+def test_multimodal_content_persists_text_and_attachment_ref_without_payload(manager):
     sid = "sess-" + uuid.uuid4().hex[:8]
     _make_session(sid)
 
+    upload_id = "a" * 32 + ".png"
     multimodal = [
         {"type": "text", "text": "what is this?"},
         {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
     ]
-    msgs = [ChatMessage(role="user", content=multimodal)]
+    msgs = [ChatMessage(
+        role="user",
+        content=multimodal,
+        metadata={
+            "attachments": [{
+                "id": upload_id,
+                "name": "diagram.png",
+                "mime": "image/png",
+                "size": 4,
+                "checksum_sha256": "sha256-digest",
+            }]
+        },
+    )]
     assert manager.replace_messages(sid, msgs) is True
 
+    expected = (
+        "what is this?\n"
+        "[1 inline media payload omitted]\n"
+        f"[Attachment: diagram.png | id={upload_id} | mime=image/png | "
+        "size=4 bytes | sha256=sha256-digest]"
+    )
+
+    db = _TS()
+    try:
+        stored = db.query(cdb.ChatMessage).filter_by(session_id=sid).one()
+        assert stored.content == expected
+        assert "data:image/png;base64,AAAA" not in stored.content
+        assert "base64" not in stored.content
+        assert "AAAA" not in stored.content
+    finally:
+        db.close()
+
     # Drop the in-memory cache so the next read hydrates from the DB.
     manager.sessions.clear()
     reloaded = manager.get_session(sid)
     assert len(reloaded.history) == 1
-    # Content must come back as the original list, not a repr string blob.
-    assert reloaded.history[0].content == multimodal
+    persisted = reloaded.history[0].content
+    assert isinstance(persisted, str)
+    assert persisted == expected
+    assert reloaded.history[0].metadata["attachments"][0]["id"] == upload_id
+    assert (
+        reloaded.history[0].metadata["attachments"][0]["checksum_sha256"]
+        == "sha256-digest"
+    )
 
 
-def test_plain_string_content_still_round_trips(manager):
+def test_jsonlike_plain_string_content_still_round_trips(manager):
     sid = "sess-" + uuid.uuid4().hex[:8]
     _make_session(sid)
-    msgs = [ChatMessage(role="user", content="just text")]
+    text = '[{"type": "object", "name": "foo"}]'
+    msgs = [ChatMessage(role="user", content=text)]
     assert manager.replace_messages(sid, msgs) is True
     manager.sessions.clear()
     reloaded = manager.get_session(sid)
-    assert reloaded.history[0].content == "just text"
+    assert isinstance(reloaded.history[0].content, str)
+    assert reloaded.history[0].content == text
 
 
 def test_replace_messages_keeps_history_alias_for_context_messages(manager):

tests/test_replace_messages_upload_reservations.py

@@ -0,0 +1,259 @@
+"""Upload lifecycle guarantees for compaction's replace_messages path."""
+
+import concurrent.futures
+import json
+import os
+import threading
+import uuid
+
+import pytest
+from sqlalchemy import event
+
+import core.database as cdb
+import core.session_manager as session_manager_module
+from core.models import ChatMessage
+from src.upload_handler import UploadHandler
+from tests.helpers.sqlite_db import make_temp_sqlite
+
+
+OLD_TIMESTAMP = "2000-01-01T00:00:00"
+
+
+@pytest.fixture
+def manager_db(monkeypatch):
+    SessionLocal, engine, tmpfile = make_temp_sqlite(cdb.Base.metadata)
+    monkeypatch.setattr(session_manager_module, "SessionLocal", SessionLocal)
+    manager = session_manager_module.SessionManager.__new__(
+        session_manager_module.SessionManager
+    )
+    manager.sessions = {}
+    manager.upload_handler = None
+    try:
+        yield manager, SessionLocal, engine
+    finally:
+        engine.dispose()
+        tmpfile.close()
+        try:
+            os.unlink(tmpfile.name)
+        except OSError:
+            pass
+
+
+def _seed_session(SessionLocal, *, owner="alice", content="existing durable history"):
+    session_id = "replace-" + uuid.uuid4().hex
+    db = SessionLocal()
+    try:
+        db.add(cdb.Session(
+            id=session_id,
+            owner=owner,
+            name="Compaction reservation regression",
+            model="test-model",
+            endpoint_url="http://localhost:11434",
+            archived=False,
+            message_count=1,
+        ))
+        db.add(cdb.ChatMessage(
+            id="message-" + uuid.uuid4().hex,
+            session_id=session_id,
+            role="user",
+            content=content,
+            meta_data=json.dumps({"source": "before-replacement"}),
+        ))
+        db.commit()
+    finally:
+        db.close()
+    return session_id
+
+
+def _attachment_message(upload_id, text):
+    return ChatMessage(
+        role="user",
+        content=text,
+        metadata={
+            "attachments": [{
+                "id": upload_id,
+                "name": f"{text}.txt",
+                "mime": "text/plain",
+                "size": len(text),
+            }]
+        },
+    )
+
+
+def _durable_messages(SessionLocal, session_id):
+    db = SessionLocal()
+    try:
+        return [
+            (message.role, message.content, message.meta_data)
+            for message in db.query(cdb.ChatMessage)
+            .filter(cdb.ChatMessage.session_id == session_id)
+            .order_by(cdb.ChatMessage.timestamp, cdb.ChatMessage.id)
+            .all()
+        ]
+    finally:
+        db.close()
+
+
+def test_replace_messages_reserves_every_incoming_attachment_before_delete(
+    manager_db,
+    monkeypatch,
+):
+    manager, SessionLocal, engine = manager_db
+    session_id = _seed_session(SessionLocal)
+    manager.upload_handler = object()
+    incoming = [
+        _attachment_message("1" * 32 + ".txt", "first"),
+        _attachment_message("2" * 32 + ".txt", "second"),
+    ]
+    message_mutations = []
+    reservation_calls = []
+
+    def record_sql(_conn, _cursor, statement, _parameters, _context, _executemany):
+        normalized = statement.lstrip().upper()
+        if normalized.startswith(("DELETE FROM CHAT_MESSAGES", "INSERT INTO CHAT_MESSAGES")):
+            message_mutations.append(normalized.split(maxsplit=1)[0])
+
+    def reserve(handler, owner, content, metadata):
+        assert message_mutations == []
+        reservation_calls.append((handler, owner, content, metadata))
+        return None
+
+    event.listen(engine, "before_cursor_execute", record_sql)
+    monkeypatch.setattr(
+        session_manager_module,
+        "reserve_message_upload_references",
+        reserve,
+    )
+    try:
+        assert manager.replace_messages(session_id, incoming) is True
+    finally:
+        event.remove(engine, "before_cursor_execute", record_sql)
+
+    assert [call[2] for call in reservation_calls] == ["first", "second"]
+    assert all(call[0] is manager.upload_handler for call in reservation_calls)
+    assert all(call[1] == "alice" for call in reservation_calls)
+    assert message_mutations == ["DELETE", "INSERT"]
+
+
+def test_replace_messages_reservation_failure_leaves_durable_history_unchanged(
+    manager_db,
+    monkeypatch,
+):
+    manager, SessionLocal, engine = manager_db
+    session_id = _seed_session(SessionLocal)
+    before = _durable_messages(SessionLocal, session_id)
+    manager.upload_handler = object()
+    missing_upload_id = "4" * 32 + ".txt"
+    incoming = [
+        _attachment_message("3" * 32 + ".txt", "available"),
+        _attachment_message(missing_upload_id, "missing"),
+    ]
+    calls = []
+    message_mutations = []
+
+    def record_sql(_conn, _cursor, statement, _parameters, _context, _executemany):
+        normalized = statement.lstrip().upper()
+        if normalized.startswith(("DELETE FROM CHAT_MESSAGES", "INSERT INTO CHAT_MESSAGES")):
+            message_mutations.append(normalized.split(maxsplit=1)[0])
+
+    def reserve(_handler, _owner, content, _metadata):
+        calls.append(content)
+        return missing_upload_id if content == "missing" else None
+
+    event.listen(engine, "before_cursor_execute", record_sql)
+    monkeypatch.setattr(
+        session_manager_module,
+        "reserve_message_upload_references",
+        reserve,
+    )
+    try:
+        assert manager.replace_messages(session_id, incoming) is False
+    finally:
+        event.remove(engine, "before_cursor_execute", record_sql)
+
+    assert calls == ["available", "missing"]
+    assert message_mutations == []
+    assert _durable_messages(SessionLocal, session_id) == before
+    assert [message.content for message in manager.sessions[session_id].history] == [
+        "existing durable history"
+    ]
+    assert all("_db_id" not in (message.metadata or {}) for message in incoming)
+
+
+def test_cleanup_cannot_delete_attachment_during_concurrent_compaction_replacement(
+    manager_db,
+    monkeypatch,
+    tmp_path,
+):
+    manager, SessionLocal, _engine = manager_db
+    session_id = _seed_session(SessionLocal)
+    base_dir = tmp_path / "base"
+    upload_dir = tmp_path / "uploads"
+    base_dir.mkdir()
+    upload_dir.mkdir()
+    handler = UploadHandler(str(base_dir), str(upload_dir))
+    manager.upload_handler = handler
+
+    upload_id = "5" * 32 + ".txt"
+    upload_hash = "6" * 64
+    dated_dir = upload_dir / "2000" / "01" / "01"
+    dated_dir.mkdir(parents=True)
+    upload_path = dated_dir / upload_id
+    upload_path.write_text("attachment retained by compaction", encoding="utf-8")
+    upload_row = {
+        "id": upload_id,
+        "path": str(upload_path),
+        "mime": "text/plain",
+        "size": upload_path.stat().st_size,
+        "name": "compaction.txt",
+        "original_name": "compaction.txt",
+        "hash": upload_hash,
+        "checksum_sha256": upload_hash,
+        "uploaded_at": OLD_TIMESTAMP,
+        "created_at": OLD_TIMESTAMP,
+        "last_accessed": OLD_TIMESTAMP,
+        "owner": "alice",
+    }
+    (upload_dir / "uploads.json").write_text(
+        json.dumps({f"alice:{upload_hash}": upload_row}),
+        encoding="utf-8",
+    )
+    handler._index_cache = None
+
+    reservation_write_entered = threading.Event()
+    release_reservation_write = threading.Event()
+    real_atomic_write = handler._atomic_write_json
+
+    def block_reservation_write(path, data, *, sync_backup=False):
+        refreshed = any(
+            isinstance(row, dict)
+            and row.get("id") == upload_id
+            and row.get("last_accessed") != OLD_TIMESTAMP
+            for row in data.values()
+        )
+        if sync_backup and refreshed and not reservation_write_entered.is_set():
+            reservation_write_entered.set()
+            assert release_reservation_write.wait(5)
+        return real_atomic_write(path, data, sync_backup=sync_backup)
+
+    monkeypatch.setattr(handler, "_atomic_write_json", block_reservation_write)
+    incoming = [_attachment_message(upload_id, "retained after compaction")]
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        replace_future = pool.submit(manager.replace_messages, session_id, incoming)
+        assert reservation_write_entered.wait(5)
+        cleanup_future = pool.submit(handler.cleanup_old_uploads, set(), set())
+        try:
+            with pytest.raises(concurrent.futures.TimeoutError):
+                cleanup_future.result(timeout=0.1)
+        finally:
+            release_reservation_write.set()
+
+        assert replace_future.result(timeout=5) is True
+        assert cleanup_future.result(timeout=5) == 0
+
+    assert upload_path.is_file()
+    assert handler.resolve_upload(upload_id, owner="alice") is not None
+    durable = _durable_messages(SessionLocal, session_id)
+    assert len(durable) == 1
+    assert json.loads(durable[0][2])["attachments"][0]["id"] == upload_id

tests/test_upload_handler_cleanup.py

@@ -0,0 +1,831 @@
+import asyncio
+import concurrent.futures
+import json
+import os
+import threading
+from datetime import datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+
+from core.database import (
+    Base,
+    ChatMessage as DbChatMessage,
+    CalendarCal,
+    CalendarEvent,
+    Document,
+    DocumentVersion,
+    GalleryImage,
+    Note,
+    Session as DbSession,
+)
+from src.upload_handler import (
+    UploadCleanupSafetyError,
+    UploadHandler,
+    extract_internal_upload_ids,
+    reserve_message_upload_references,
+    reserve_upload_references,
+)
+from tests.helpers.sqlite_db import make_temp_sqlite
+
+
+OLD_TIMESTAMP = "2000-01-01T00:00:00"
+
+
+class _AdminAuth:
+    is_configured = True
+
+    @staticmethod
+    def is_admin(user):
+        return user == "admin"
+
+
+class _AdminRequest:
+    headers = {}
+    state = SimpleNamespace(current_user="admin")
+    app = SimpleNamespace(state=SimpleNamespace(auth_manager=_AdminAuth()))
+
+
+def _make_handler(tmp_path: Path) -> UploadHandler:
+    base_dir = tmp_path / "base"
+    upload_dir = tmp_path / "uploads"
+    base_dir.mkdir()
+    upload_dir.mkdir()
+    return UploadHandler(str(base_dir), str(upload_dir))
+
+
+def _seed_old_uploads(handler: UploadHandler, rows: list[dict]) -> dict[str, Path]:
+    dated_dir = Path(handler.upload_dir) / "2000" / "01" / "01"
+    dated_dir.mkdir(parents=True)
+    index = {}
+    paths = {}
+    for row in rows:
+        upload_id = row["id"]
+        path = dated_dir / upload_id
+        path.write_bytes(row.get("bytes", upload_id.encode("ascii")))
+        info = {
+            "id": upload_id,
+            "path": str(path),
+            "mime": row.get("mime", "application/octet-stream"),
+            "size": path.stat().st_size,
+            "name": row.get("name", upload_id),
+            "original_name": row.get("name", upload_id),
+            "hash": row["hash"],
+            "checksum_sha256": row["hash"],
+            "uploaded_at": row.get("uploaded_at", OLD_TIMESTAMP),
+            "created_at": row.get("created_at", OLD_TIMESTAMP),
+            "last_accessed": row.get("last_accessed", OLD_TIMESTAMP),
+            "owner": row.get("owner", "alice"),
+        }
+        index[f"{info['owner']}:{info['hash']}"] = info
+        paths[upload_id] = path
+
+    Path(handler.upload_dir, "uploads.json").write_text(
+        json.dumps(index),
+        encoding="utf-8",
+    )
+    handler._index_cache = None
+    return paths
+
+
+def _manual_cleanup_endpoint(handler: UploadHandler, monkeypatch):
+    import fastapi.dependencies.utils as dependency_utils
+    from routes.upload_routes import router, setup_upload_routes
+
+    monkeypatch.setattr(dependency_utils, "ensure_multipart_is_installed", lambda: None)
+    before = len(router.routes)
+    setup_upload_routes(handler)
+    return {
+        route.endpoint.__name__: route.endpoint
+        for route in router.routes[before:]
+    }["manual_cleanup"]
+
+
+def _reference_database(monkeypatch, *, upload_id: str, gallery_hash: str = None):
+    from routes import upload_routes
+
+    SessionLocal, engine, tmpfile = make_temp_sqlite(Base.metadata)
+    db = SessionLocal()
+    try:
+        db.add(DbSession(
+            id="session-1",
+            name="Cleanup regression",
+            endpoint_url="http://localhost",
+            model="test-model",
+            owner="alice",
+        ))
+        db.add(DbChatMessage(
+            id="message-1",
+            session_id="session-1",
+            role="user",
+            content=f"[Attachment: retained.png | id={upload_id} | mime=image/png]",
+            meta_data=json.dumps({
+                "attachments": [{
+                    "id": upload_id,
+                    "name": "retained.png",
+                    "mime": "image/png",
+                    "size": 8,
+                }]
+            }),
+        ))
+        if gallery_hash:
+            db.add(GalleryImage(
+                id="gallery-cleanup-reference",
+                filename="abcdef123456.png",
+                prompt="Chat upload",
+                owner="alice",
+                file_hash=gallery_hash,
+            ))
+        db.commit()
+    finally:
+        db.close()
+
+    monkeypatch.setattr(upload_routes, "SessionLocal", SessionLocal)
+    return engine, tmpfile
+
+
+def test_admin_cleanup_preserves_referenced_upload_and_reconciles_deleted_row(
+    tmp_path,
+    monkeypatch,
+):
+    handler = _make_handler(tmp_path)
+    referenced_id = "a" * 32 + ".png"
+    unreferenced_id = "b" * 32 + ".txt"
+    gallery_id = "7" * 32 + ".png"
+    gallery_hash = "7" * 64
+    paths = _seed_old_uploads(handler, [
+        {
+            "id": referenced_id,
+            "hash": "1" * 64,
+            "mime": "image/png",
+        },
+        {
+            "id": unreferenced_id,
+            "hash": "2" * 64,
+            "mime": "text/plain",
+        },
+        {
+            "id": gallery_id,
+            "hash": gallery_hash,
+            "mime": "image/png",
+        },
+    ])
+    engine, tmpfile = _reference_database(
+        monkeypatch,
+        upload_id=referenced_id,
+        gallery_hash=gallery_hash,
+    )
+
+    try:
+        response = asyncio.run(
+            _manual_cleanup_endpoint(handler, monkeypatch)(_AdminRequest())
+        )
+    finally:
+        engine.dispose()
+        tmpfile.close()
+        try:
+            os.unlink(tmpfile.name)
+        except OSError:
+            pass
+
+    assert response == {"status": "success", "files_cleaned": 1}
+    assert paths[referenced_id].is_file()
+    referenced_info = handler.get_upload_info(referenced_id)
+    assert referenced_info is not None
+    assert handler.resolve_upload(referenced_id, owner="alice") is not None
+    assert paths[gallery_id].is_file()
+    assert handler.get_upload_info(gallery_id) is not None
+
+    assert not paths[unreferenced_id].exists()
+    assert handler.get_upload_info(unreferenced_id) is None
+    assert handler.resolve_upload(unreferenced_id, owner="alice") is None
+
+    live_index = json.loads(
+        Path(handler.upload_dir, "uploads.json").read_text(encoding="utf-8")
+    )
+    assert {info["id"] for info in live_index.values()} == {
+        referenced_id,
+        gallery_id,
+    }
+    backup_index = json.loads(
+        Path(handler.upload_dir, "uploads.json.bak").read_text(encoding="utf-8")
+    )
+    assert {info["id"] for info in backup_index.values()} == {
+        referenced_id,
+        gallery_id,
+    }
+
+    # Recovery must not resurrect the deliberately deleted row.
+    Path(handler.upload_dir, "uploads.json").write_text("{broken", encoding="utf-8")
+    handler._index_cache = None
+    assert handler.get_upload_info(unreferenced_id) is None
+    assert paths[referenced_id].parent.is_dir()
+
+
+def test_cleanup_retains_upload_and_all_rows_when_index_rows_disagree(tmp_path):
+    handler = _make_handler(tmp_path)
+    upload_id = "c" * 32 + ".txt"
+    path = _seed_old_uploads(handler, [{
+        "id": upload_id,
+        "hash": "1" * 64,
+        "mime": "text/plain",
+        "owner": "alice",
+    }])[upload_id]
+    index_path = Path(handler.upload_dir, "uploads.json")
+    index = json.loads(index_path.read_text(encoding="utf-8"))
+    alice_row = next(iter(index.values()))
+    index["bob:" + "2" * 64] = {
+        **alice_row,
+        "owner": "bob",
+        "hash": "2" * 64,
+        "checksum_sha256": "2" * 64,
+    }
+    index_path.write_text(json.dumps(index), encoding="utf-8")
+    handler._index_cache = None
+
+    assert handler.cleanup_old_uploads(set(), set()) == 0
+    assert path.is_file()
+    assert json.loads(index_path.read_text(encoding="utf-8")) == index
+
+
+def test_cleanup_retains_lone_row_without_authoritative_lifecycle_metadata(tmp_path):
+    handler = _make_handler(tmp_path)
+    upload_id = "6" * 32 + ".txt"
+    path = _seed_old_uploads(handler, [{
+        "id": upload_id,
+        "hash": "6" * 64,
+        "mime": "text/plain",
+    }])[upload_id]
+    index_path = Path(handler.upload_dir, "uploads.json")
+    index = json.loads(index_path.read_text(encoding="utf-8"))
+    row = next(iter(index.values()))
+    for field in (
+        "owner",
+        "hash",
+        "checksum_sha256",
+        "uploaded_at",
+        "created_at",
+        "last_accessed",
+    ):
+        row.pop(field)
+    index_path.write_text(json.dumps(index), encoding="utf-8")
+    handler._index_cache = None
+
+    assert handler.cleanup_old_uploads(set(), set()) == 0
+    assert path.is_file()
+    assert json.loads(index_path.read_text(encoding="utf-8")) == index
+
+
+def test_reservation_and_cleanup_are_serialized_without_dangling_references(
+    tmp_path,
+    monkeypatch,
+):
+    # Writer wins: reservation holds the shared index lock, refreshes access,
+    # then cleanup observes the refreshed row and preserves the file.
+    writer_root = tmp_path / "writer-wins"
+    writer_root.mkdir()
+    writer_handler = _make_handler(writer_root)
+    upload_id = "2" * 32 + ".txt"
+    writer_path = _seed_old_uploads(writer_handler, [{
+        "id": upload_id,
+        "hash": "2" * 64,
+        "mime": "text/plain",
+    }])[upload_id]
+    write_entered = threading.Event()
+    release_write = threading.Event()
+    real_atomic_write = writer_handler._atomic_write_json
+
+    def blocking_reservation_write(path, data, *, sync_backup=False):
+        refreshed = any(
+            isinstance(row, dict) and row.get("last_accessed") != OLD_TIMESTAMP
+            for row in data.values()
+        )
+        if sync_backup and refreshed and not write_entered.is_set():
+            write_entered.set()
+            assert release_write.wait(5)
+        return real_atomic_write(path, data, sync_backup=sync_backup)
+
+    monkeypatch.setattr(writer_handler, "_atomic_write_json", blocking_reservation_write)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        reserve_future = pool.submit(
+            writer_handler.reserve_upload,
+            upload_id,
+            owner="alice",
+        )
+        assert write_entered.wait(5)
+        cleanup_future = pool.submit(writer_handler.cleanup_old_uploads, set(), set())
+        release_write.set()
+        assert reserve_future.result(timeout=5) is not None
+        assert cleanup_future.result(timeout=5) == 0
+    assert writer_path.is_file()
+
+    # Cleanup wins: reservation cannot pass the same lock until the row and
+    # bytes are gone, then fails so a caller cannot commit a dangling reference.
+    cleanup_root = tmp_path / "cleanup-wins"
+    cleanup_root.mkdir()
+    cleanup_handler = _make_handler(cleanup_root)
+    cleanup_path = _seed_old_uploads(cleanup_handler, [{
+        "id": upload_id,
+        "hash": "3" * 64,
+        "mime": "text/plain",
+    }])[upload_id]
+    remove_entered = threading.Event()
+    release_remove = threading.Event()
+    real_remove = os.remove
+
+    def blocking_remove(candidate):
+        if os.path.realpath(candidate) == os.path.realpath(cleanup_path):
+            remove_entered.set()
+            assert release_remove.wait(5)
+        return real_remove(candidate)
+
+    monkeypatch.setattr("src.upload_handler.os.remove", blocking_remove)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
+        cleanup_future = pool.submit(cleanup_handler.cleanup_old_uploads, set(), set())
+        assert remove_entered.wait(5)
+        reserve_future = pool.submit(
+            cleanup_handler.reserve_upload,
+            upload_id,
+            owner="alice",
+        )
+        release_remove.set()
+        assert cleanup_future.result(timeout=5) == 1
+        assert reserve_future.result(timeout=5) is None
+    assert not cleanup_path.exists()
+
+
+def test_admin_cleanup_reference_discovery_failure_returns_503_without_deleting(
+    tmp_path,
+    monkeypatch,
+):
+    from routes import upload_routes
+
+    handler = _make_handler(tmp_path)
+    upload_id = "d" * 32 + ".png"
+    path = _seed_old_uploads(handler, [
+        {"id": upload_id, "hash": "4" * 64, "mime": "image/png"},
+    ])[upload_id]
+
+    def fail_reference_scan():
+        raise RuntimeError("database unavailable")
+
+    monkeypatch.setattr(
+        upload_routes,
+        "_collect_persisted_upload_references",
+        fail_reference_scan,
+    )
+    endpoint = _manual_cleanup_endpoint(handler, monkeypatch)
+
+    with pytest.raises(HTTPException) as exc:
+        asyncio.run(endpoint(_AdminRequest()))
+
+    assert exc.value.status_code == 503
+    assert path.is_file()
+    assert handler.get_upload_info(upload_id) is not None
+
+
+def test_cleanup_restores_index_when_file_removal_fails(tmp_path, monkeypatch):
+    handler = _make_handler(tmp_path)
+    upload_id = "e" * 32 + ".txt"
+    path = _seed_old_uploads(handler, [
+        {
+            "id": upload_id,
+            "hash": "5" * 64,
+            "mime": "text/plain",
+        },
+    ])[upload_id]
+
+    real_remove = os.remove
+
+    def fail_target_remove(candidate):
+        if os.path.realpath(candidate) == os.path.realpath(path):
+            raise PermissionError("file is in use")
+        return real_remove(candidate)
+
+    monkeypatch.setattr("src.upload_handler.os.remove", fail_target_remove)
+
+    assert handler.cleanup_old_uploads(set(), set()) == 0
+    assert path.is_file()
+    assert handler.get_upload_info(upload_id) is not None
+    assert any(
+        info["id"] == upload_id
+        for info in json.loads(
+            Path(handler.upload_dir, "uploads.json").read_text(encoding="utf-8")
+        ).values()
+    )
+    assert any(
+        info["id"] == upload_id
+        for info in json.loads(
+            Path(handler.upload_dir, "uploads.json.bak").read_text(encoding="utf-8")
+        ).values()
+    )
+
+
+def test_admin_cleanup_with_corrupt_index_returns_503_and_fails_closed(
+    tmp_path,
+    monkeypatch,
+):
+    from routes import upload_routes
+
+    handler = _make_handler(tmp_path)
+    upload_id = "9" * 32 + ".png"
+    path = _seed_old_uploads(handler, [
+        {"id": upload_id, "hash": "9" * 64, "mime": "image/png"},
+    ])[upload_id]
+    Path(handler.upload_dir, "uploads.json").write_text(
+        '{"alice:broken": {',
+        encoding="utf-8",
+    )
+    handler._index_cache = None
+
+    monkeypatch.setattr(
+        upload_routes,
+        "_collect_persisted_upload_references",
+        lambda: (set(), set()),
+    )
+    endpoint = _manual_cleanup_endpoint(handler, monkeypatch)
+
+    with pytest.raises(HTTPException) as exc:
+        asyncio.run(endpoint(_AdminRequest()))
+
+    assert exc.value.status_code == 503
+    assert path.is_file()
+
+
+def test_cleanup_with_missing_live_index_fails_closed(tmp_path):
+    handler = _make_handler(tmp_path)
+    upload_id = "8" * 32 + ".png"
+    dated_dir = Path(handler.upload_dir) / "2000" / "01" / "01"
+    dated_dir.mkdir(parents=True)
+    path = dated_dir / upload_id
+    path.write_bytes(b"unindexed bytes")
+
+    with pytest.raises(UploadCleanupSafetyError):
+        handler.cleanup_old_uploads(set(), set())
+
+    assert path.is_file()
+
+
+def test_reference_discovery_covers_all_durable_upload_stores(
+    monkeypatch,
+):
+    from routes import upload_routes
+
+    document_id = "f" * 32 + ".pdf"
+    version_id = "1" * 32 + ".pdf"
+    note_upload_id = "3" * 32 + ".png"
+    note_color_id = "2" * 32 + ".png"
+    calendar_upload_id = "4" * 32 + ".png"
+    event_upload_id = "5" * 32 + ".png"
+    event_description_id = "7" * 32 + ".txt"
+    event_location_id = "8" * 32 + ".png"
+    gallery_hash = "6" * 64
+    SessionLocal, engine, tmpfile = make_temp_sqlite(Base.metadata)
+    db = SessionLocal()
+    try:
+        db.add(DbSession(
+            id="session-2",
+            name="Reference sources",
+            endpoint_url="http://localhost",
+            model="test-model",
+            owner="alice",
+        ))
+        db.add(Document(
+            id="document-1",
+            session_id="session-2",
+            title="PDF",
+            current_content=f'<!-- pdf_source upload_id="{document_id}" -->',
+            owner="alice",
+        ))
+        db.add(DocumentVersion(
+            id="version-1",
+            document_id="document-1",
+            version_number=1,
+            content=f'<!-- pdf_form_source upload_id="{version_id}" fields="1" -->',
+        ))
+        db.add(GalleryImage(
+            id="gallery-1",
+            # Gallery filenames are normally generated 12-hex names, so this
+            # record proves retention comes from its stored content hash.
+            filename="abcdef123456.png",
+            prompt="Chat upload",
+            owner="alice",
+            file_hash=gallery_hash,
+        ))
+        db.add(Note(
+            id="note-1",
+            owner="alice",
+            title="Photo note",
+            image_url=f"/api/upload/{note_upload_id}",
+            color=f"odysseus://attachment/{note_color_id}",
+        ))
+        db.add(CalendarCal(
+            id="calendar-1",
+            owner="alice",
+            name="Personal",
+            color=f"/api/upload/{calendar_upload_id}",
+        ))
+        db.add(CalendarEvent(
+            uid="event-1",
+            calendar_id="calendar-1",
+            summary="Photo event",
+            dtstart=datetime(2026, 7, 10, 12, 0),
+            dtend=datetime(2026, 7, 10, 13, 0),
+            color=f"/api/upload/{event_upload_id}",
+            description=f"Notes: odysseus://attachment/{event_description_id}",
+            location=f"/api/upload/{event_location_id}",
+        ))
+        db.commit()
+    finally:
+        db.close()
+
+    monkeypatch.setattr(upload_routes, "SessionLocal", SessionLocal)
+    try:
+        referenced_ids, referenced_hashes = (
+            upload_routes._collect_persisted_upload_references()
+        )
+    finally:
+        engine.dispose()
+        tmpfile.close()
+        try:
+            os.unlink(tmpfile.name)
+        except OSError:
+            pass
+
+    assert {
+        document_id,
+        version_id,
+        note_upload_id,
+        note_color_id,
+        calendar_upload_id,
+        event_upload_id,
+        event_description_id,
+        event_location_id,
+    } <= referenced_ids
+    assert gallery_hash in referenced_hashes
+
+
+def test_write_reservation_extracts_only_explicit_internal_references():
+    upload_id = "a" * 32 + ".png"
+    checksum_like_text = "b" * 32
+
+    assert extract_internal_upload_ids(checksum_like_text) == set()
+    assert extract_internal_upload_ids(f"sha={checksum_like_text}") == set()
+    assert extract_internal_upload_ids({
+        "image": f"/api/upload/{upload_id}",
+        "nested": [f"odysseus://attachment/{upload_id}"],
+    }) == {upload_id}
+    assert extract_internal_upload_ids(
+        f'<!-- pdf_source upload_id="{upload_id}" -->'
+    ) == {upload_id}
+    assert extract_internal_upload_ids(
+        f"[Attachment: photo.png | id={upload_id} | mime=image/png]"
+    ) == {upload_id}
+    extensionless_id = "c" * 32
+    assert extract_internal_upload_ids(
+        f"See /api/upload/{extensionless_id}. Then continue."
+    ) == {extensionless_id}
+    assert extract_internal_upload_ids(
+        f"Attachment: odysseus://attachment/{extensionless_id}: ready"
+    ) == {extensionless_id}
+    assert extract_internal_upload_ids(f"/api/upload/{upload_id}/extra") == set()
+
+
+def test_reservation_never_uses_admin_override(tmp_path):
+    handler = _make_handler(tmp_path)
+    upload_id = "c" * 32 + ".txt"
+    _seed_old_uploads(handler, [{
+        "id": upload_id,
+        "hash": "c" * 64,
+        "mime": "text/plain",
+        "owner": "alice",
+    }])
+
+    assert reserve_upload_references(
+        handler,
+        "alice",
+        f"odysseus://attachment/{upload_id}",
+    ) is None
+    assert reserve_upload_references(
+        handler,
+        "admin",
+        f"odysseus://attachment/{upload_id}",
+    ) == upload_id
+    assert handler.reserve_upload(
+        upload_id,
+        owner="admin",
+        auth_manager=_AdminAuth(),
+        allow_admin=False,
+    ) is None
+    assert reserve_message_upload_references(
+        handler,
+        "admin",
+        "legacy attachment metadata",
+        {"attachments": [{"id": upload_id, "name": "owned.txt"}]},
+    ) == upload_id
+
+
+def test_remaining_durable_writers_reserve_before_commit(monkeypatch):
+    import core.database as database
+    import core.session_manager as session_manager_module
+    import src.database as legacy_database
+    from core.models import ChatMessage
+    from core.session_manager import SessionManager
+    from src import tool_utils
+    from src.agent_tools.document_tools import EditDocumentTool
+    from src.tools.calendar import do_manage_calendar
+    from src.tools.notes import do_manage_notes
+
+    upload_id = "6" * 32 + ".png"
+
+    class RejectingHandler:
+        def reserve_upload(self, _candidate, **_kwargs):
+            return None
+
+    handler = RejectingHandler()
+    monkeypatch.setattr(tool_utils, "_upload_handler", handler)
+
+    SessionLocal, engine, tmpfile = make_temp_sqlite(Base.metadata)
+    monkeypatch.setattr(database, "SessionLocal", SessionLocal)
+    monkeypatch.setattr(legacy_database, "SessionLocal", SessionLocal)
+    monkeypatch.setattr(legacy_database, "Document", Document, raising=False)
+    monkeypatch.setattr(
+        legacy_database,
+        "DocumentVersion",
+        DocumentVersion,
+        raising=False,
+    )
+    monkeypatch.setattr(session_manager_module, "SessionLocal", SessionLocal)
+    db = SessionLocal()
+    try:
+        db.add(DbSession(
+            id="writer-session",
+            name="Writer coverage",
+            endpoint_url="http://localhost",
+            model="test-model",
+            owner="alice",
+        ))
+        db.add(Document(
+            id="email-document",
+            session_id="writer-session",
+            title="New Email",
+            language="email",
+            current_content="To: team@example.test\nSubject: Status\n---\nOld body",
+            version_count=1,
+            owner="alice",
+        ))
+        db.commit()
+
+        manager = SessionManager()
+        manager.upload_handler = handler
+        manager._persist_message(
+            "writer-session",
+            ChatMessage(
+                "user",
+                "attachment",
+                metadata={"attachments": [{"id": upload_id}]},
+            ),
+        )
+
+        document_result = asyncio.run(EditDocumentTool().execute(
+            "<<<FIND>>>\n\n<<<REPLACE>>>\n"
+            f"See /api/upload/{upload_id}\n<<<END>>>",
+            {"doc_id": "email-document", "owner": "alice"},
+        ))
+        assert document_result["exit_code"] == 1
+        assert "no longer available" in document_result["error"]
+
+        calendar_result = asyncio.run(do_manage_calendar(
+            json.dumps({
+                "action": "create_event",
+                "summary": "Attachment review",
+                "dtstart": "2026-07-12T12:00:00",
+                "description": f"See /api/upload/{upload_id}",
+            }),
+            owner="alice",
+        ))
+        assert calendar_result["exit_code"] == 1
+        assert "no longer available" in calendar_result["error"]
+
+        note_result = asyncio.run(do_manage_notes(
+            json.dumps({
+                "action": "add",
+                "title": "Attachment note",
+                "content": f"See /api/upload/{upload_id}",
+            }),
+            owner="alice",
+        ))
+        assert note_result["exit_code"] == 1
+        assert "no longer available" in note_result["error"]
+
+        verify = SessionLocal()
+        try:
+            assert verify.query(DbChatMessage).count() == 0
+            stored_doc = verify.query(Document).filter(Document.id == "email-document").one()
+            assert stored_doc.current_content.endswith("Old body")
+            assert verify.query(CalendarEvent).count() == 0
+            assert verify.query(Note).count() == 0
+        finally:
+            verify.close()
+    finally:
+        db.close()
+        engine.dispose()
+        tmpfile.close()
+        try:
+            os.unlink(tmpfile.name)
+        except OSError:
+            pass
+
+
+def test_note_calendar_and_document_routes_reserve_before_database_writes(monkeypatch):
+    from routes.calendar_routes import EventCreate, setup_calendar_routes
+    from routes import document_routes
+    from routes.document_helpers import DocumentCreate
+    from routes.note_routes import NoteCreate, setup_note_routes
+    from src import auth_helpers
+
+    upload_id = "d" * 32 + ".png"
+
+    class RejectingHandler:
+        def __init__(self):
+            self.calls = []
+
+        def reserve_upload(self, candidate, **kwargs):
+            self.calls.append((candidate, kwargs))
+            return None
+
+    request = SimpleNamespace(
+        state=SimpleNamespace(current_user="alice", api_token=False),
+        app=SimpleNamespace(state=SimpleNamespace()),
+    )
+
+    note_handler = RejectingHandler()
+    note_router = setup_note_routes(upload_handler=note_handler)
+    create_note = next(
+        route.endpoint for route in note_router.routes
+        if route.endpoint.__name__ == "create_note"
+    )
+    with pytest.raises(HTTPException) as note_error:
+        create_note(
+            request,
+            NoteCreate(image_url=f"/api/upload/{upload_id}"),
+        )
+    assert note_error.value.status_code == 409
+    assert note_handler.calls == [
+        (upload_id, {"owner": "alice", "allow_admin": False})
+    ]
+
+    calendar_handler = RejectingHandler()
+    calendar_router = setup_calendar_routes(upload_handler=calendar_handler)
+    create_event = next(
+        route.endpoint for route in calendar_router.routes
+        if route.endpoint.__name__ == "create_event"
+    )
+    with pytest.raises(HTTPException) as calendar_error:
+        asyncio.run(create_event(
+            request,
+            EventCreate(
+                summary="Photo",
+                dtstart="2026-07-10T12:00:00",
+                color=f"odysseus://attachment/{upload_id}",
+            ),
+        ))
+    assert calendar_error.value.status_code == 409
+    assert calendar_handler.calls == [
+        (upload_id, {"owner": "alice", "allow_admin": False})
+    ]
+
+    class EmptyDb:
+        @staticmethod
+        def close():
+            return None
+
+    document_handler = RejectingHandler()
+    monkeypatch.setattr(document_routes, "SessionLocal", EmptyDb)
+    monkeypatch.setattr(
+        auth_helpers,
+        "require_privilege",
+        lambda _request, _privilege: "alice",
+    )
+    document_router = document_routes.setup_document_routes(
+        SimpleNamespace(),
+        document_handler,
+    )
+    create_document = next(
+        route.endpoint for route in document_router.routes
+        if route.endpoint.__name__ == "create_document"
+    )
+    with pytest.raises(HTTPException) as document_error:
+        asyncio.run(create_document(
+            request,
+            DocumentCreate(
+                language="markdown",
+                content=f"![image](/api/upload/{upload_id})",
+            ),
+        ))
+    assert document_error.value.status_code == 409
+    assert document_handler.calls == [
+        (upload_id, {"owner": "alice", "allow_admin": False})
+    ]