fix(mcp_manager): remove timeout from MCP connection attempts and handle registration cleanup

* fix(db): restrict data/app.db to 0600

app.db holds bearer-token hashes, bcrypt password hashes, and encrypted
provider keys but was created under the default umask (0644 -> world-readable),
unlike .app_key/vault/integrations which are already 0600 via safe_chmod.

init_db() now chmods the SQLite file to 0600 right after create_all (POSIX
only; no-op on Windows, skipped for Postgres / in-memory). Unconditional and
idempotent, so it also re-locks already-deployed 0644 installs on next
startup. The transient rollback journal inherits 0600 from the parent file at
creation - no sidecar handling needed; -wal/-shm don't exist until WAL is
enabled (#4409 C4) and inherit the same mode then.

Satisfies Rule B, unblocking #4413 and the vault/integration secret moves.
Mirrors src/secret_storage.py:43-45.

Verified: security + DB-permission suites pass; 6 pre-existing visual_report
failures (missing markdown/nh3 deps) are unrelated.

Closes #4407

* fix(db): harden SQLite path parsing and re-lock sidecars

Address review feedback on #4420.

P2: derive the file to chmod from engine.url (SQLAlchemy's parsed URL)
via _sqlite_db_path(), instead of DATABASE_URL.replace("sqlite:///", "").
A driver-qualified URL (sqlite+pysqlite://) or one carrying query args
(?cache=shared) previously slipped past the prefix check / string slice
and left the DB world-readable; the parsed path resolves correctly and
drops the query.

P3: re-lock stale -wal/-shm/-journal sidecars to 0o600 at startup. The
main file is chmod'd first, so any sidecar SQLite creates afterward
inherits 0o600, but a -wal/-shm left world-readable by an older 0o644
install (once WAL was enabled) could still expose DB pages. Absent
sidecars are the normal case, not an error.

Tests: unit-test _sqlite_db_path across driver/query/memory/postgres URL
forms, and a subprocess test asserting stale 0o644 -wal/-shm are
re-locked on startup.

* fix(db): handle sqlite file URI app db permissions

* fix(db): close remaining SQLite permission bypasses

---------

Co-authored-by: Ethan <23321960+0xLeathery@users.noreply.github.com>
Co-authored-by: Alexandre Teixeira <alexandremagteixeira@gmail.com>

.github/scripts/focused_test_guidance.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Report focused pytest guidance for changed paths under tests/."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+from collections.abc import Iterable
+from pathlib import PurePosixPath
+
+
+def parse_paths(raw_paths: bytes) -> list[str]:
+    """Decode the NUL-delimited output of ``git diff --name-only -z``."""
+    return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path]
+
+
+def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]:
+    """Return changed ``tests/`` paths using GitHub PR three-dot semantics.
+
+    GitHub PR changed files are based on the merge base and the PR head, not a
+    direct endpoint diff between the current base branch tip and the PR head.
+    Using the direct endpoint diff can include files changed only on the base
+    branch when the PR branch is stale.
+    """
+    merge_base = subprocess.check_output(
+        ["git", "merge-base", base_sha, head_sha],
+        stderr=subprocess.DEVNULL,
+    ).strip()
+    raw_paths = subprocess.check_output(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "--diff-filter=ACMRT",
+            "-z",
+            os.fsdecode(merge_base),
+            head_sha,
+            "--",
+            "tests/",
+        ],
+    )
+    return parse_paths(raw_paths)
+
+
+def select_test_paths(paths: Iterable[str]) -> list[str]:
+    """Return unique, repository-relative paths contained by tests/."""
+    selected: set[str] = set()
+    for raw_path in paths:
+        path = PurePosixPath(raw_path)
+        if path.is_absolute() or ".." in path.parts:
+            continue
+        parts = tuple(part for part in path.parts if part != ".")
+        if len(parts) >= 2 and parts[0] == "tests":
+            selected.add(PurePosixPath(*parts).as_posix())
+    return sorted(selected)
+
+
+def is_pytest_file(path: str) -> bool:
+    """Return whether a changed path follows this repository's pytest naming."""
+    name = PurePosixPath(path).name
+    return name.endswith(".py") and (
+        name.startswith("test_") or name.endswith("_test.py")
+    )
+
+
+def pytest_command(paths: Iterable[str]) -> str:
+    """Build a copyable pytest command for changed runnable test files."""
+    command = ["python3", "-m", "pytest", "-q", *paths]
+    return shlex.join(command)
+
+
+def format_report(paths: Iterable[str]) -> str:
+    """Format focused guidance for CI logs and the workflow summary."""
+    changed_paths = select_test_paths(paths)
+    runnable_paths = [path for path in changed_paths if is_pytest_file(path)]
+    lines = ["## Focused test guidance (report-only)", ""]
+    if not changed_paths:
+        lines.append("No changed paths under `tests/`.")
+    else:
+        lines.extend(["Changed paths under `tests/`:", ""])
+        lines.extend(f"- `{path}`" for path in changed_paths)
+    lines.extend(["", "Suggested focused validation:", ""])
+    if runnable_paths:
+        lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```")
+    else:
+        lines.append("No directly runnable pytest files changed.")
+    lines.extend(
+        [
+            "",
+            "This guidance does not infer tests from source changes. "
+            "Existing blocking CI remains the source of truth.",
+        ]
+    )
+    return "\n".join(lines)
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Report focused pytest guidance for changed tests/ paths.",
+    )
+    parser.add_argument("--base-sha", help="Pull request base commit SHA.")
+    parser.add_argument("--head-sha", help="Pull request head commit SHA.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(sys.argv[1:] if argv is None else argv)
+    if bool(args.base_sha) != bool(args.head_sha):
+        raise SystemExit("--base-sha and --head-sha must be provided together")
+
+    if args.base_sha and args.head_sha:
+        paths = changed_paths_from_merge_base(args.base_sha, args.head_sha)
+    else:
+        paths = parse_paths(sys.stdin.buffer.read())
+
+    print(format_report(paths))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/ci.yml

@@ -15,6 +15,60 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  focused-test-guidance:
+    name: Focused test guidance (report-only)
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Report changed test paths
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          report_file="$RUNNER_TEMP/focused-test-guidance.md"
+          publish_report() {
+            cat "$report_file"
+            if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+              cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true
+            fi
+            return 0
+          }
+
+          report_unavailable() {
+            {
+              printf '%s\n\n' '## Focused test guidance unavailable (report-only)'
+              printf '%s\n\n' "$1"
+              printf '%s\n' 'Existing blocking CI remains the source of truth.'
+            } > "$report_file"
+            publish_report
+            exit 0
+          }
+
+          if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
+            report_unavailable "Pull request base/head metadata is missing."
+          fi
+
+          if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
+            report_unavailable "The pull request base commit is unavailable locally."
+          fi
+
+          if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then
+            report_unavailable "The pull request head commit is unavailable locally."
+          fi
+
+          if ! python3 .github/scripts/focused_test_guidance.py \
+            --base-sha "$BASE_SHA" \
+            --head-sha "$HEAD_SHA" > "$report_file"; then
+            report_unavailable "The focused test guidance helper could not produce a report."
+          fi
+
+          publish_report
+
   python-syntax:
     name: Python syntax (compileall)
     runs-on: ubuntu-latest

ROADMAP.md

@@ -12,7 +12,6 @@ the codebase, you are probably right to stay away.
   and WSL all need coverage.
 
 - Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden. 
-- Self-host troubleshooting cookbook. Document the weird 30-second fixes that otherwise become 30-minute searches: Dovecot cleartext auth for local stacks, ntfy Android Instant Delivery for non-ntfy.sh servers, clipboard limits on plain-HTTP Tailscale URLs, Radicale collection URLs, and similar traps.
 - Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments.
 - Cookbook SGLang support across platforms. Make sure SGLang setup/serve works
   predictably on Linux, Windows/WSL, macOS where possible, Docker, and common

app.py

@@ -3,6 +3,7 @@ import mimetypes
 import os
 import sys
 import asyncio
+import time
 
 # On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
 # When started via `python -m uvicorn` from a terminal, uvicorn sets this
@@ -204,12 +205,43 @@ class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
         path = request.url.path or ""
         if not should_track_interactive_request(path, request.method):
             return await call_next(request)
+        async def _stop_background():
+            try:
+                await task_scheduler.stop_background_tasks_for_foreground(reason=f"foreground request {request.method} {path}")
+            except Exception:
+                logging.getLogger("app.foreground_gate").debug("foreground task stop failed", exc_info=True)
+        asyncio.create_task(_stop_background())
         async with track_interactive_request(path, request.method):
             return await call_next(request)
 
 
+class _SlowRequestLogMiddleware(_BaseHTTPMiddleware):
+    async def dispatch(self, request, call_next):
+        start = time.perf_counter()
+        status = 500
+        try:
+            response = await call_next(request)
+            status = getattr(response, "status_code", 0) or 0
+            return response
+        finally:
+            elapsed = time.perf_counter() - start
+            try:
+                threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75")
+            except Exception:
+                threshold = 0.75
+            if elapsed >= threshold:
+                logging.getLogger("app.slow_request").warning(
+                    "slow_request method=%s path=%s status=%s elapsed=%.3fs",
+                    request.method,
+                    request.url.path,
+                    status,
+                    elapsed,
+                )
+
+
 app.add_middleware(_RequestTimeoutMiddleware)
 app.add_middleware(_InteractiveActivityMiddleware)
+app.add_middleware(_SlowRequestLogMiddleware)
 
 # ========= AUTH =========
 from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
@@ -600,6 +632,12 @@ app.include_router(auth_router)
 async def activity_heartbeat():
     from src.interactive_gate import mark_browser_activity
     await mark_browser_activity()
+    async def _stop_background():
+        try:
+            await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
+        except Exception:
+            logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
+    asyncio.create_task(_stop_background())
     return {"ok": True}
 
 
@@ -617,7 +655,12 @@ app.include_router(setup_emoji_routes())
 # Sessions
 from routes.session_routes import setup_session_routes
 session_config = {"REQUEST_TIMEOUT": REQUEST_TIMEOUT, "OPENAI_API_KEY": OPENAI_API_KEY, "SESSIONS_FILE": SESSIONS_FILE}
-app.include_router(setup_session_routes(session_manager, session_config, webhook_manager=webhook_manager))
+app.include_router(setup_session_routes(
+    session_manager,
+    session_config,
+    webhook_manager=webhook_manager,
+    upload_handler=upload_handler,
+))
 
 # Admin Danger Zone wipes (Settings → System → Danger Zone)
 from routes.admin_wipe_routes import setup_admin_wipe_routes
@@ -645,8 +688,8 @@ from routes.research.research_routes import setup_research_routes
 app.include_router(setup_research_routes(research_handler, session_manager=session_manager))
 
 # History
-from routes.history_routes import setup_history_routes
+from routes.history.history_routes import setup_history_routes
-app.include_router(setup_history_routes(session_manager))
+app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler))
 
 # Search
 from routes.search_routes import setup_search_routes
@@ -725,7 +768,7 @@ app.include_router(setup_assistant_routes(task_scheduler))
 
 # Calendar (CalDAV)
 from routes.calendar_routes import setup_calendar_routes
-calendar_router = setup_calendar_routes()
+calendar_router = setup_calendar_routes(upload_handler=upload_handler)
 app.include_router(calendar_router)
 
 # Shell (user-facing command execution)
@@ -788,7 +831,7 @@ logger.info("Webhook & API token routes initialized")
 
 # Notes (Google Keep-style notes/todos)
 from routes.note_routes import setup_note_routes
-app.include_router(setup_note_routes(task_scheduler))
+app.include_router(setup_note_routes(task_scheduler, upload_handler=upload_handler))
 
 # Email
 from routes.email_routes import setup_email_routes
@@ -813,7 +856,7 @@ from routes.vault_routes import setup_vault_routes
 app.include_router(setup_vault_routes())
 
 # Contacts (CardDAV)
-from routes.contacts_routes import setup_contacts_routes
+from routes.contacts.contacts_routes import setup_contacts_routes
 app.include_router(setup_contacts_routes())
 
 from companion import setup_companion_routes
@@ -889,6 +932,34 @@ async def get_version():
 async def health_check() -> Dict[str, str]:
     return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()}
 
+@app.post("/api/client-perf")
+async def client_perf(request: Request):
+    """Low-volume frontend timing reports for stalls that happen before SSE logs."""
+    try:
+        data = await request.json()
+    except Exception:
+        data = {}
+    try:
+        kind = str(data.get("type") or "client").replace("\n", " ")[:80]
+        total_ms = float(data.get("total_ms") or 0)
+        stages = data.get("stages") if isinstance(data.get("stages"), list) else []
+        stage_txt = " ".join(
+            f"{str(s.get('name') or '')[:40]}={float(s.get('delta_ms') or 0):.0f}ms"
+            for s in stages[:20]
+            if isinstance(s, dict)
+        )
+        extra = str(data.get("extra") or "").replace("\n", " ")[:200]
+        logging.getLogger("app.client_perf").warning(
+            "client_perf type=%s total=%.0fms %s%s",
+            kind,
+            total_ms,
+            stage_txt,
+            f" extra={extra}" if extra else "",
+        )
+    except Exception:
+        logging.getLogger("app.client_perf").debug("client_perf log failed", exc_info=True)
+    return {"ok": True}
+
 @app.get("/api/ready")
 async def readiness_check() -> JSONResponse:
     """Readiness / integrity self-check — DB, data dir, local-first storage.
@@ -977,7 +1048,7 @@ async def _startup_event():
         except BaseException as e:
             logger.warning(f"Built-in MCP registration failed (non-critical): {type(e).__name__}: {e}")
         try:
-            await asyncio.wait_for(mcp_manager.connect_all_enabled(), timeout=20)
+            await mcp_manager.connect_all_enabled()
         except asyncio.TimeoutError:
             logger.warning("User MCP startup timed out (non-critical)")
         except BaseException as e:
@@ -985,45 +1056,43 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
 
-    # Pre-warm the RAG tool index off the request path. Loading the local
+    # Startup warmups are opt-in. They make later requests a little warmer, but
-    # embedding model + opening ChromaDB + indexing the built-in tools is a
+    # they also compete with the first seconds of real UI use on slow or busy
-    # one-time ~1-3s cost that otherwise lands on the user's FIRST message
+    # machines. Default to clear/idle startup and let requests warm what they use.
-    # (showing up as a big `tool_selection` time). Doing it here makes the
+    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
-    # first turn as fast as subsequent ones (warm embed ≈ a few ms).
+    if _startup_warmups_enabled:
-    async def _warmup_tool_index():
+        async def _warmup_tool_index():
-        try:
+            try:
-            from src.tool_index import get_tool_index
+                from src.tool_index import get_tool_index
-            idx = await asyncio.to_thread(get_tool_index)
+                idx = await asyncio.to_thread(get_tool_index)
-            if idx:
+                if idx:
-                await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
+                    await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
-                logger.info("[startup] Tool index pre-warmed")
+                    logger.info("[startup] Tool index pre-warmed")
-        except Exception as e:
+            except Exception as e:
-            logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
+                logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
 
-    _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
+        _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
-    # Warmup: ping all known LLM endpoints to prime connections
-    async def _warmup_endpoints():
-        try:
-            import httpx
-            # model_discovery has no get_endpoints(); that call raised
-            # AttributeError every run and silently disabled warmup/keepalive.
-            # Resolve the /models probe URLs via the real discovery API, off the
-            # event loop since discovery does a blocking port scan.
-            urls = (
-                await asyncio.to_thread(model_discovery.warmup_ping_urls)
-                if model_discovery else []
-            )
-            for url in urls:
-                try:
-                    async with httpx.AsyncClient(timeout=5.0) as client:
-                        await client.get(url)
-                    logger.info(f"Warmup ping OK: {url}")
-                except Exception as e:
-                    logger.debug(f"Warmup ping failed for endpoint: {e}")
-        except Exception as e:
-            logger.debug(f"Warmup ping skipped: {e}")
 
-    _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
+        async def _warmup_endpoints():
+            try:
+                import httpx
+                urls = (
+                    await asyncio.to_thread(model_discovery.warmup_ping_urls)
+                    if model_discovery else []
+                )
+                for url in urls:
+                    try:
+                        async with httpx.AsyncClient(timeout=5.0) as client:
+                            await client.get(url)
+                        logger.info(f"Warmup ping OK: {url}")
+                    except Exception as e:
+                        logger.debug(f"Warmup ping failed for endpoint: {e}")
+            except Exception as e:
+                logger.debug(f"Warmup ping skipped: {e}")
+
+        _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
+    else:
+        logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
 
     # Keep-alive is opt-in. The ping path performs model discovery, and when
     # stale LAN endpoints are configured it can add periodic backend pressure

core/database.py

@@ -3,13 +3,16 @@ import logging
 import sqlite3
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import Optional
+from urllib.parse import unquote, urlparse
 from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
 from sqlalchemy.orm import relationship, sessionmaker, backref
 
 from src.runtime_paths import get_app_root
+from core.platform_compat import safe_chmod, IS_WINDOWS
 
 logger = logging.getLogger(__name__)
 
@@ -42,12 +45,28 @@ def _default_database_url() -> str:
 
 
 def _normalize_sqlite_url(url: str) -> str:
-    if not url.startswith("sqlite:///"):
+    """Resolve relative ordinary SQLite paths without rewriting URI filenames."""
+    try:
+        parsed = make_url(url)
+    except Exception:
         return url
-    db_path = url.replace("sqlite:///", "", 1)
+
-    if db_path == ":memory:" or os.path.isabs(db_path):
+    if parsed.get_backend_name() != "sqlite":
         return url
-    return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}"
+
+    db_path = parsed.database
+    if (
+        not db_path
+        or db_path == ":memory:"
+        or str(db_path).lower().startswith("file:")
+        or os.path.isabs(str(db_path))
+    ):
+        return url
+
+    absolute_path = (Path(get_app_root()) / str(db_path)).resolve().as_posix()
+    return parsed.set(database=absolute_path).render_as_string(
+        hide_password=False
+    )
 
 
 # Get database URL from environment, default to SQLite in DATA_DIR
@@ -59,6 +78,59 @@ engine = create_engine(
     connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
 )
 
+
+# Sidecar files SQLite can create next to the main DB. -journal is the default
+# rollback journal; -wal/-shm appear once WAL is enabled. Each can hold copies of
+# secret-bearing pages, so they get the same 0o600 lockdown as the DB itself.
+_SQLITE_SIDECARS = ("-journal", "-wal", "-shm")
+
+
+def _sqlite_db_path(url) -> Optional[str]:
+    """Return the filesystem path for a file-backed SQLite URL.
+
+    SQLite query parameters such as ``mode=memory`` only affect filename
+    semantics when SQLAlchemy enables URI handling with ``uri=true``. Ordinary
+    file URLs must therefore remain file-backed even when they contain a query
+    parameter named ``mode``.
+
+    For SQLite ``file:`` URIs, an empty authority or ``localhost`` identifies a
+    local path. Other authorities are retained as UNC-style paths.
+    """
+    if url.get_backend_name() != "sqlite":
+        return None
+
+    db_path = url.database
+    if not db_path or db_path == ":memory:":
+        return None
+
+    db_path = str(db_path)
+    query = {
+        str(key).lower(): str(value).strip().lower()
+        for key, value in dict(getattr(url, "query", {}) or {}).items()
+    }
+    uri_enabled = query.get("uri") in {"1", "true", "yes", "on"}
+    is_file_uri = db_path.lower().startswith("file:")
+
+    if not uri_enabled or not is_file_uri:
+        return db_path
+
+    if (
+        db_path.lower().startswith("file::memory:")
+        or query.get("mode") == "memory"
+    ):
+        return None
+
+    parsed = urlparse(db_path)
+    fs_path = parsed.path or ""
+    if not fs_path or fs_path == ":memory:":
+        return None
+
+    authority = parsed.netloc
+    if authority and authority.lower() != "localhost":
+        fs_path = f"//{authority}{fs_path}"
+
+    return unquote(fs_path)
+
 # Create session factory
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
@@ -1819,6 +1891,41 @@ def init_db():
     """
     _migrate_model_endpoints()
     Base.metadata.create_all(bind=engine)
+    # Lock the DB file (and any SQLite sidecars) to 0o600 — it holds bearer-token
+    # + bcrypt hashes and encrypted provider keys. POSIX only; safe_chmod no-ops
+    # on Windows (ACL-restricted profile dir) and the path helper returns None for
+    # Postgres / in-memory. Must stay AFTER create_all: the file is born here at
+    # the umask default, and nothing below resets the mode. The path comes from
+    # engine.url (SQLAlchemy's parsed URL), so a driver-qualified or query-tagged
+    # DATABASE_URL still resolves to the real file instead of slipping through.
+    db_path = _sqlite_db_path(engine.url)
+    if db_path is not None:
+        # Fail closed-loud on the main file: this is the only access control on
+        # it, so if the chmod genuinely fails (read-only FS, foreign owner) an
+        # operator should hear about it. safe_chmod also returns False as a
+        # Windows no-op, so guard on IS_WINDOWS to avoid a spurious warning there.
+        if not safe_chmod(db_path, 0o600) and not IS_WINDOWS:
+            logger.warning(
+                "Could not restrict %s to 0o600; it holds secrets and may be "
+                "world-readable. Check filesystem permissions and ownership.",
+                db_path,
+            )
+        # Re-lock any sidecars present at startup. New ones inherit the main
+        # file's mode (now 0o600, since we set it first), and they're usually
+        # absent here, but a stale -wal/-shm/-journal left by an older 0o644
+        # install could still expose secret pages. Absent sidecars are the
+        # normal case, not an error — only a failed chmod warrants a warning.
+        for suffix in _SQLITE_SIDECARS:
+            sidecar = db_path + suffix
+            if (
+                os.path.exists(sidecar)
+                and not safe_chmod(sidecar, 0o600)
+                and not IS_WINDOWS
+            ):
+                logger.warning(
+                    "Could not restrict %s to 0o600; it may expose DB pages.",
+                    sidecar,
+                )
     _migrate_add_hidden_models_column()
     _migrate_add_cached_models_column()
     _migrate_add_pinned_models_column()
@@ -1904,6 +2011,20 @@ def _migrate_chat_messages_fts():
     conn = None
     try:
         conn = sqlite3.connect(db_path)
+        fts_content_expr_new = (
+            "CASE WHEN instr(COALESCE(new.content, ''), ';base64,') > 0 "
+            "OR instr(COALESCE(new.content, ''), 'data:image/') > 0 "
+            "OR instr(COALESCE(new.content, ''), 'data:audio/') > 0 "
+            "THEN '[inline media omitted from search index]' "
+            "ELSE COALESCE(new.content, '') END"
+        )
+        fts_content_expr_cm = (
+            "CASE WHEN instr(COALESCE(cm.content, ''), ';base64,') > 0 "
+            "OR instr(COALESCE(cm.content, ''), 'data:image/') > 0 "
+            "OR instr(COALESCE(cm.content, ''), 'data:audio/') > 0 "
+            "THEN '[inline media omitted from search index]' "
+            "ELSE COALESCE(cm.content, '') END"
+        )
         try:
             conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS temp._odysseus_fts5_probe USING fts5(content)")
             conn.execute("DROP TABLE IF EXISTS temp._odysseus_fts5_probe")
@@ -1912,7 +2033,7 @@ def _migrate_chat_messages_fts():
             return
 
         conn.executescript(
-            """
+            f"""
             CREATE VIRTUAL TABLE IF NOT EXISTS chat_messages_fts USING fts5(
                 content,
                 message_id UNINDEXED,
@@ -1920,10 +2041,14 @@ def _migrate_chat_messages_fts():
                 role UNINDEXED
             );
 
+            DROP TRIGGER IF EXISTS chat_messages_fts_ai;
+            DROP TRIGGER IF EXISTS chat_messages_fts_ad;
+            DROP TRIGGER IF EXISTS chat_messages_fts_au;
+
             CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ai
             AFTER INSERT ON chat_messages BEGIN
                 INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-                VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
+                VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role);
             END;
 
             CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ad
@@ -1935,14 +2060,14 @@ def _migrate_chat_messages_fts():
             AFTER UPDATE ON chat_messages BEGIN
                 DELETE FROM chat_messages_fts WHERE message_id = old.id;
                 INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-                VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
+                VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role);
             END;
             """
         )
         conn.execute(
-            """
+            f"""
             INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-            SELECT COALESCE(cm.content, ''), cm.id, cm.session_id, cm.role
+            SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
             FROM chat_messages cm
             WHERE NOT EXISTS (
                 SELECT 1 FROM chat_messages_fts fts
@@ -1950,6 +2075,7 @@ def _migrate_chat_messages_fts():
             )
             """
         )
+        _scrub_legacy_chat_message_fts_media(conn)
         conn.commit()
     except Exception as e:
         logging.getLogger(__name__).warning(f"chat_messages FTS migration failed: {e}")
@@ -1960,6 +2086,37 @@ def _migrate_chat_messages_fts():
             pass
 
 
+def _scrub_legacy_chat_message_fts_media(conn) -> None:
+    """Replace already-indexed inline media rows with searchable text only."""
+    try:
+        from src.attachment_refs import search_index_text
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"chat_messages FTS media scrub skipped: {e}")
+        return
+
+    try:
+        rows = conn.execute(
+            """
+            SELECT id, session_id, role, content
+            FROM chat_messages
+            WHERE instr(COALESCE(content, ''), ';base64,') > 0
+               OR instr(COALESCE(content, ''), 'data:image/') > 0
+               OR instr(COALESCE(content, ''), 'data:audio/') > 0
+            """
+        ).fetchall()
+        for message_id, session_id, role, content in rows:
+            conn.execute("DELETE FROM chat_messages_fts WHERE message_id = ?", (message_id,))
+            conn.execute(
+                """
+                INSERT INTO chat_messages_fts(content, message_id, session_id, role)
+                VALUES (?, ?, ?, ?)
+                """,
+                (search_index_text(content), message_id, session_id, role),
+            )
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"chat_messages FTS media scrub failed: {e}")
+
+
 def _migrate_add_email_smtp_security():
     """Add explicit SMTP security mode for Proton Bridge/custom local SMTP."""
     import sqlite3

core/session_manager.py

@@ -16,6 +16,8 @@ from typing import Dict, Optional
 
 from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive
 from .models import Session, ChatMessage
+from src.attachment_refs import persistable_message_content
+from src.upload_handler import reserve_message_upload_references
 
 # Re-export singleton accessors from models for convenience
 from .models import set_session_manager_instance, get_session_manager_instance
@@ -72,6 +74,7 @@ class SessionManager:
     def __init__(self, sessions_file: str = None):
         # sessions_file kept for backward compat, not used
         self.sessions: Dict[str, Session] = {}
+        self.upload_handler = None
         self.load_sessions()
 
     # ------------------------------------------------------------------
@@ -230,17 +233,26 @@ class SessionManager:
                 logger.warning("Dropping message for deleted session %s", session_id)
                 return
 
+            missing_upload_id = reserve_message_upload_references(
+                getattr(self, "upload_handler", None),
+                getattr(db_session, "owner", None),
+                message.content,
+                message.metadata,
+            )
+            if missing_upload_id:
+                raise ValueError(
+                    f"Referenced upload is no longer available: {missing_upload_id}"
+                )
+
             msg_id = str(uuid.uuid4())
             msg_time = datetime.utcnow()
             if message.metadata is None:
                 message.metadata = {}
             message.metadata.setdefault('timestamp', _message_timestamp_iso(msg_time))
-            # Multimodal content (image/audio attachments) is a list — serialize
+            # Multimodal content may contain provider data URLs for the live
-            # to JSON so the Text column can store it.  On reload, _db_to_session
+            # model call. Persist only readable text plus attachment references
-            # detects the JSON-array prefix and parses it back.
+            # so chat_messages/FTS do not duplicate upload bytes.
-            _content = message.content
+            _content = persistable_message_content(message.content, message.metadata)
-            if isinstance(_content, list):
-                _content = json.dumps(_content)
             db_message = DbChatMessage(
                 id=msg_id,
                 session_id=session_id,
@@ -322,6 +334,28 @@ class SessionManager:
         session = self.get_session(session_id)
         db = SessionLocal()
         try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if db_session is None:
+                logger.warning("Cannot replace history for missing session %s", session_id)
+                return False
+
+            # Reserve every incoming attachment before removing any durable
+            # message row. reserve_upload() shares the upload lifecycle lock
+            # with cleanup, so an upload cannot be deleted between this
+            # ownership check/access touch and the replacement transaction.
+            # A failed reservation must leave the existing transcript intact.
+            for message in messages:
+                missing_upload_id = reserve_message_upload_references(
+                    getattr(self, "upload_handler", None),
+                    getattr(db_session, "owner", None),
+                    message.content,
+                    message.metadata,
+                )
+                if missing_upload_id:
+                    raise ValueError(
+                        f"Referenced upload is no longer available: {missing_upload_id}"
+                    )
+
             db.query(DbChatMessage).filter(DbChatMessage.session_id == session_id).delete()
             now = datetime.now(timezone.utc)
             for i, message in enumerate(messages):
@@ -330,15 +364,9 @@ class SessionManager:
                     id=msg_id,
                     session_id=session_id,
                     role=message.role,
-                    # Multimodal content (image/audio attachments) is a list;
+                    # Mirrors _persist_message: keep raw media bytes out of the
-                    # serialize to JSON so the Text column round-trips via
+                    # persisted transcript and search index.
-                    # _parse_msg_content. Storing the raw list let SQLAlchemy
+                    content=persistable_message_content(message.content, message.metadata),
-                    # bind its single-quoted repr, which _parse_msg_content
-                    # cannot parse (it looks for double-quoted "type"), so the
-                    # attachment was destroyed on reload. Mirrors _persist_message.
-                    content=(json.dumps(message.content)
-                             if isinstance(message.content, list)
-                             else message.content),
                     meta_data=json.dumps(message.metadata) if message.metadata else None,
                     timestamp=now + timedelta(microseconds=i),
                 )
@@ -347,12 +375,10 @@ class SessionManager:
                     message.metadata = {}
                 message.metadata["_db_id"] = msg_id
 
-            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            db_session.message_count = len(messages)
-            if db_session:
+            db_session.updated_at = now
-                db_session.message_count = len(messages)
+            db_session.last_accessed = now
-                db_session.updated_at = now
+            db_session.last_message_at = now
-                db_session.last_accessed = now
-                db_session.last_message_at = now
 
             db.commit()
             session.history = list(messages)

docs/attachments.md

@@ -0,0 +1,85 @@
+# Attachment References and Upload Storage
+
+Odysseus stores uploaded bytes once under the configured upload directory and
+passes stable references through chat history, tools, and future artifact work.
+The goal is to avoid duplicating large inline media payloads in
+`chat_messages.content` or the SQLite FTS index.
+
+## Reference Shape
+
+Attachment references use this minimum shape:
+
+```json
+{
+  "type": "attachment_ref",
+  "attachment_id": "32hex-or-32hex.ext",
+  "name": "original-filename.png",
+  "mime": "image/png",
+  "size": 12345,
+  "checksum_sha256": "hex-digest",
+  "created_at": "2026-07-09T12:00:00"
+}
+```
+
+Optional fields such as `width`, `height`, `vision`, `vision_model`, and
+`gallery_id` may be present when the uploader or preprocessing path knows them.
+
+## Persistence
+
+The live model call may still receive provider-specific multimodal blocks for
+the current turn. Persistence is different:
+
+- `chat_messages.content` stores readable text plus compact attachment reference
+  lines, never raw `data:*;base64,...` upload bytes.
+- `chat_messages.metadata.attachments` stores structured attachment reference
+  metadata for UI reloads and future processing.
+- The SQLite FTS migration recreates chat-message FTS triggers so new rows do
+  not index inline media payloads, and it scrubs legacy rows that were already
+  indexed with data URLs.
+
+## Tool Access
+
+Agent/tool context receives upload entries as `attachment_ref` manifests with an
+`odysseus://attachment/<id>` URI and `read_policy: "owner_checked_upload"`.
+
+For compatibility with existing built-in tools, a local `path` may be included
+only after all of these checks pass:
+
+- the upload ID resolves through `UploadHandler.resolve_upload`;
+- the requested owner is allowed to read the upload;
+- the file remains inside the configured upload directory;
+- the file path is inside the tool-readable roots.
+
+External MCP/custom tools should treat the URI and attachment ID as the stable
+contract and request bytes through an owner-checked server path, not by assuming
+host filesystem layout.
+
+## Retention and Deletion
+
+Current retention behavior is conservative:
+
+- uploads are indexed in `uploads.json` with owner, checksum, MIME type, size,
+  and creation time;
+- admin cleanup first scans persisted chat metadata/content, document versions,
+  PDF source markers, gallery hashes, notes, and calendar records for live
+  references;
+- cleanup fails closed if that reference scan cannot complete, and the lower-level
+  cleanup API removes nothing unless it receives a complete reference snapshot;
+- expired, unreferenced uploads are removed during the completed scan, while
+  attachment-bearing writers must first take an owner-checked reservation that
+  serializes with deletion and refreshes the upload's access timestamp;
+- deliberate removal atomically drops matching `uploads.json` rows before deleting
+  the bytes and restores those rows if filesystem removal fails;
+- deleting a chat removes the chat rows but does not immediately delete shared
+  upload bytes, because the same upload may also be referenced by gallery items,
+  documents, duplicate-upload rows, or future artifact records.
+
+There is no distinct artifact table in the current schema. Artifact-like upload
+references persisted in chat or document text are covered by the canonical
+attachment-ID scan; any future artifact store must be added to reference discovery
+before cleanup is allowed to consider its uploads unreferenced.
+
+Cleanup and write reservations share the upload-index lock. This closes the
+scan/write/delete race in the documented single-worker deployment; a future
+multi-process deployment must add an inter-process lock or move lifecycle state
+into the database before enabling destructive cleanup in more than one worker.

docs/setup.md

@@ -472,4 +472,4 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum
 `memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
 
 To back up or restore everything in `data/`, see the
-[Backup & Restore guide](docs/backup-restore.md).
+[Backup & Restore guide](backup-restore.md).

mcp_servers/email_server.py

@@ -58,6 +58,11 @@ def _uid_fetch_rows(data) -> list:
 _ACCOUNT_CACHE: dict = {}  # key = normalized account selector -> config dict
 _MCP_OWNER_ARG = "_odysseus_owner"
 _CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None)
+_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_EMAIL_OWNER", "ODYSSEUS_EMAIL_OWNER")
+_OWNER_SCOPE_ERROR = (
+    "Error: email MCP requires an authenticated owner or ODYSSEUS_MCP_EMAIL_OWNER "
+    "when owner-scoped email accounts are configured."
+)
 
 
 def _clean_header_value(value) -> str:
@@ -71,13 +76,29 @@ def _db_path() -> Path:
     return Path(APP_DB)
 
 
+def _configured_owner() -> str | None:
+    for key in _OWNER_ENV_KEYS:
+        owner = os.environ.get(key, "").strip()
+        if owner:
+            return owner
+    return None
+
+
 def _current_owner() -> str:
     owner = _CURRENT_OWNER.get()
-    return str(owner or "").strip()
+    return str(owner or _configured_owner() or "").strip()
+
+
+def _account_owner(row: dict) -> str:
+    return str(row.get("owner") or "").strip()
+
+
+def _has_owner_scoped_accounts(rows: list[dict]) -> bool:
+    return any(_account_owner(r) for r in rows)
 
 
 def _account_visible_to_owner(row: dict, owner: str) -> bool:
-    row_owner = str(row.get("owner") or "").strip()
+    row_owner = _account_owner(row)
     if row_owner == owner:
         return True
     if row_owner:
@@ -96,8 +117,7 @@ def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]:
     if owner:
         return [r for r in rows if _account_visible_to_owner(r, owner)]
 
-    owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()}
+    if _has_owner_scoped_accounts(rows):
-    if len(owners) > 1:
         return []
     return rows
 
@@ -106,8 +126,7 @@ def _mcp_owner_required(rows: list[dict] | None = None) -> bool:
     if _current_owner():
         return False
     rows = rows if rows is not None else _read_accounts_from_db()
-    owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()}
+    return _has_owner_scoped_accounts(rows)
-    return len(owners) > 1
 
 
 def _load_email_writing_style() -> str:
@@ -274,6 +293,8 @@ def _load_config(account: str | None = None) -> dict:
     }
 
     raw_rows = _read_accounts_from_db()
+    if _mcp_owner_required(raw_rows):
+        raise ValueError(_OWNER_SCOPE_ERROR)
     rows = _filter_accounts_for_owner(raw_rows)
     row = _resolve_account_from_rows(rows, account)
     if _current_owner() and raw_rows and not rows:
@@ -1193,10 +1214,14 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b
     UI. This closes the auto-send hole that let earlier models invent
     signatures and ship them to real recipients without confirmation."""
     if _read_agent_email_confirm_setting():
+        # Even confirmation-first sends must resolve the selected account now.
+        # Otherwise a caller could stage a pending draft against another
+        # owner's account selector before browser approval handles it.
+        cfg = _load_config(account)
         return _stash_agent_draft(
             to=to, subject=subject, body=body,
             in_reply_to=in_reply_to, references=references,
-            cc=cc, bcc=bcc, account=account,
+            cc=cc, bcc=bcc, account=cfg.get("account_id") or account,
         )
     send_account, cfg = _resolve_send_config(account)
     msg = EmailMessage()
@@ -2142,10 +2167,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
     try:
         all_db_accounts = _read_accounts_from_db()
         if _mcp_owner_required(all_db_accounts):
-            return [TextContent(
+            return [TextContent(type="text", text=_OWNER_SCOPE_ERROR)]
-                type="text",
-                text="Error: email MCP requires an authenticated owner when multiple email account owners are configured.",
-            )]
 
         if name == "list_email_accounts":
             rows = _filter_accounts_for_owner(all_db_accounts)

requirements.txt

@@ -3,6 +3,7 @@ uvicorn
 python-multipart
 python-dotenv
 httpx
+httpcore>=1.0,<2.0
 pydantic>=2.13.4
 pydantic-settings>=2.14.1
 SQLAlchemy

routes/calendar_routes.py

@@ -13,8 +13,9 @@ from sqlalchemy import or_, and_
 from dateutil.rrule import rrulestr
 
 from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
-from src.auth_helpers import require_user
+from src.auth_helpers import effective_user, require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
+from src.upload_handler import reserve_upload_references
 
 logger = logging.getLogger(__name__)
 
@@ -697,9 +698,18 @@ def _expand_rrule(
 
 # ── Routes ──
 
-def setup_calendar_routes() -> APIRouter:
+def setup_calendar_routes(upload_handler=None) -> APIRouter:
     router = APIRouter(prefix="/api/calendar", tags=["calendar"])
 
+    def _reserve_calendar_uploads(request: Request, *values) -> None:
+        missing_id = reserve_upload_references(
+            upload_handler,
+            effective_user(request),
+            *values,
+        )
+        if missing_id:
+            raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
+
     # ── CalDAV multi-account helpers ─────────────────────────────────────────
 
     def _get_caldav_accounts(owner: str) -> list:
@@ -913,7 +923,24 @@ def setup_calendar_routes() -> APIRouter:
             '</d:prop></d:propfind>'
         )
         try:
-            async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False) as cx:
+            # Build an SSL context that trusts the operator's custom CA bundle
+            # (SSL_CERT_FILE / REQUESTS_CA_BUNDLE) so self-signed CalDAV servers
+            # pass the pre-flight the same way they pass the real sync.
+            # trust_env=False is kept to block proxy/auth env leakage; the CA
+            # bundle is loaded explicitly instead.
+            import ssl as _ssl
+            _ssl_ctx = _ssl.create_default_context()
+            # Disable VERIFY_X509_STRICT so certs without a keyUsage extension
+            # (common in self-signed setups) are accepted, matching the
+            # requests/urllib3 behavior used by the CalDAV sync path.
+            _ssl_ctx.verify_flags &= ~_ssl.VERIFY_X509_STRICT
+            _ca_bundle = _os.environ.get("SSL_CERT_FILE") or _os.environ.get("REQUESTS_CA_BUNDLE")
+            if _ca_bundle:
+                if _os.path.isfile(_ca_bundle):
+                    _ssl_ctx.load_verify_locations(_ca_bundle)
+                else:
+                    logger.warning("CalDAV test: CA bundle %s not found, using system CAs", _ca_bundle)
+            async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False, verify=_ssl_ctx) as cx:
                 r = await cx.request(
                     "PROPFIND", url,
                     auth=(user, pw),
@@ -1070,6 +1097,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.post("/events")
     async def create_event(request: Request, data: EventCreate):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, data.color, data.description, data.location)
         db = SessionLocal()
         try:
             cal = None
@@ -1131,6 +1159,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.put("/events/{uid}")
     async def update_event(request: Request, uid: str, data: EventUpdate):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, data.color, data.description, data.location)
         try:
             base_uid = _resolve_base_uid(uid)
         except ValueError as e:
@@ -1224,6 +1253,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.post("/calendars")
     async def create_calendar(request: Request, name: str = "Imported", color: str = "#5b8abf"):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, color)
         db = SessionLocal()
         try:
             cal = CalendarCal(
@@ -1246,6 +1276,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.put("/calendars/{cal_id}")
     async def update_calendar(request: Request, cal_id: str, name: str = None, color: str = None):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, color)
         db = SessionLocal()
         try:
             cal = _get_or_404_calendar(db, cal_id, owner)

routes/chat_helpers.py

@@ -14,8 +14,10 @@ from core.database import Session as DBSession, ModelEndpoint
 from src.llm_core import normalize_model_id
 from src.endpoint_resolver import normalize_base
 from src.context_compactor import maybe_compact, trim_for_context
+from src.model_context import estimate_tokens
 from src.auth_helpers import effective_user
 from src.prompt_security import untrusted_context_message
+from src.attachment_refs import attachment_ref
 from routes.prefs_routes import _load_for_user as load_prefs_for_user
 
 from fastapi import HTTPException
@@ -99,6 +101,11 @@ class ChatContext:
     uprefs: dict
     preset: PresetInfo
     preprocessed: PreprocessedMessage
+    context_trimmed: bool = False
+    context_messages_before_trim: int = 0
+    context_messages_after_trim: int = 0
+    context_tokens_before_trim: int = 0
+    context_tokens_after_trim: int = 0
     # Documents auto-created server-side during preprocess (e.g. when an
     # attached fillable PDF gets rendered into a markdown editor doc).
     # The chat route emits a doc_update SSE event for each before streaming
@@ -412,13 +419,16 @@ def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[
             except Exception:
                 path = None
 
-        manifest.append({
+        ref = attachment_ref({**info, "id": info.get("id") or str(att_id)})
-            "id": info.get("id") or str(att_id),
+        ref.update({
-            "name": info.get("name") or info.get("original_name") or str(att_id),
+            "id": ref["attachment_id"],
-            "mime": info.get("mime", ""),
+            "uri": f"odysseus://attachment/{ref['attachment_id']}",
-            "size": info.get("size", 0),
+            "read_policy": "owner_checked_upload",
+            # Transitional compatibility: existing built-in tools can still use
+            # this path, but only after owner, upload-root, and tool-root checks.
             "path": path,
         })
+        manifest.append(ref)
     return manifest
 
 
@@ -777,7 +787,12 @@ async def build_chat_context(
     messages, context_length, was_compacted = await maybe_compact(
         sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
     )
+    _before_trim_messages = len(messages)
+    _before_trim_tokens = estimate_tokens(messages)
     messages = trim_for_context(messages, context_length)
+    _after_trim_messages = len(messages)
+    _after_trim_tokens = estimate_tokens(messages)
+    _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
 
     return ChatContext(
         preface=preface,
@@ -791,6 +806,11 @@ async def build_chat_context(
         uprefs=uprefs,
         preset=preset,
         preprocessed=preprocessed,
+        context_trimmed=_context_trimmed,
+        context_messages_before_trim=_before_trim_messages,
+        context_messages_after_trim=_after_trim_messages,
+        context_tokens_before_trim=_before_trim_tokens,
+        context_tokens_after_trim=_after_trim_tokens,
         auto_opened_docs=auto_opened_docs,
         uploaded_files=uploaded_files,
     )

routes/chat_routes.py

@@ -3,6 +3,7 @@
 import asyncio
 import json
 import os
+import re
 import time
 import logging
 from datetime import datetime
@@ -40,8 +41,13 @@ from routes.chat_helpers import (
     clean_thinking_for_save,
     _enforce_chat_privileges,
 )
-from src.action_intents import classify_tool_intent as _classify_tool_intent
+from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent
-from src.tool_policy import build_effective_tool_policy
+from src.tool_policy import (
+    WEB_TOOL_NAMES,
+    build_effective_tool_policy,
+    is_web_search_explicitly_denied,
+    web_search_enabled_for_turn,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -63,6 +69,78 @@ def _stream_set(session_id: str, **fields) -> None:
     rec.update(fields)
 
 
+def _message_plain_text(content: Any) -> str:
+    if isinstance(content, list):
+        parts: List[str] = []
+        for block in content:
+            if isinstance(block, dict):
+                text = block.get("text")
+                if isinstance(text, str):
+                    parts.append(text)
+            elif isinstance(block, str):
+                parts.append(block)
+        return " ".join(parts)
+    return str(content or "")
+
+
+def _last_user_plain_text(messages: List[Dict[str, Any]]) -> str:
+    for msg in reversed(messages or []):
+        if msg.get("role") == "user":
+            return _message_plain_text(msg.get("content"))
+    return ""
+
+
+def _ensure_current_request_is_latest_user(messages: List[Dict[str, Any]], current_message: str) -> List[Dict[str, Any]]:
+    """Defensively keep detached streams grounded on the request that created them."""
+    current = str(current_message or "").strip()
+    if not current:
+        return messages
+    latest = _last_user_plain_text(messages).strip()
+    if latest == current or current in latest or latest in current:
+        return messages
+    logger.warning(
+        "[chat_stream] latest user context mismatch; appending current request for model call. latest=%r current=%r",
+        latest[:120],
+        current[:120],
+    )
+    repaired = list(messages or [])
+    repaired.append({"role": "user", "content": current})
+    return repaired
+
+
+_WEB_FOLLOWUP_RE = re.compile(
+    r"^\s*(?:(?:can|could|would|will)\s+you\s+)?"
+    r"(?:check|try\s+again|look(?:\s+now|\s+it\s+up)?|search(?:\s+now|\s+online|\s+it)?|"
+    r"do\s+it|again)\??\s*$",
+    re.I,
+)
+_RECENT_WEB_CONTEXT_RE = re.compile(
+    r"\b(?:weather|forecast|rain|raining|hourly|news|headlines|rate|exchange|currency|"
+    r"price|current|latest|search|look\s+up|online)\b",
+    re.I,
+)
+
+
+def _recent_session_text(sess, limit: int = 8, max_chars: int = 2000) -> str:
+    history = getattr(sess, "history", None) or getattr(sess, "_history", None) or []
+    chunks: List[str] = []
+    for msg in history[-limit:]:
+        content = getattr(msg, "content", None)
+        if content is None and isinstance(msg, dict):
+            content = msg.get("content")
+        text = _message_plain_text(content).strip()
+        if text:
+            chunks.append(text)
+    return " ".join(chunks)[-max_chars:]
+
+
+def _is_contextual_web_followup(message: str, sess) -> bool:
+    """Treat short retry/check replies as web lookups when recent context was web."""
+    if not message or not _WEB_FOLLOWUP_RE.search(message):
+        return False
+    return bool(_RECENT_WEB_CONTEXT_RE.search(_recent_session_text(sess)))
+
+
 def _resolve_request_workspace(request, raw_value) -> tuple:
     """Resolve the posted workspace for this request: (workspace, rejected).
 
@@ -510,6 +588,7 @@ def setup_chat_routes(
         # below). Skill extraction should only learn from real agent sessions,
         # not chats we quietly promoted for a notes/calendar intent.
         user_requested_agent = (chat_mode == "agent")
+        _search_enabled = web_search_enabled_for_turn(allow_web_search, use_web)
         # Intent auto-escalation: if the user is clearly asking the assistant
         # to create a todo, reminder, or calendar event, promote chat → agent
         # for this turn so the LLM has access to manage_notes / manage_calendar.
@@ -527,6 +606,10 @@ def setup_chat_routes(
                 _tool_intent.category,
                 _tool_intent.reason,
             )
+        elif chat_mode == "chat" and _search_enabled:
+            chat_mode = "agent"
+            auto_escalated = True
+            logger.info("chat→agent auto-escalation: search enabled")
         active_doc_id = form_data.get("active_doc_id", "").strip()
         logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}")
 
@@ -619,6 +702,20 @@ def setup_chat_routes(
                     400,
                     "No model selected for this chat. Open the model picker and choose one before sending.",
                 )
+            if (
+                chat_mode == "chat"
+                and isinstance(message, str)
+                and (not _tool_intent or not _tool_intent.needs_tools)
+                and _is_contextual_web_followup(message, sess)
+            ):
+                _tool_intent = ToolIntent(True, "web", "contextual web lookup follow-up")
+                chat_mode = "agent"
+                auto_escalated = True
+                logger.info(
+                    "chat→agent auto-escalation: category=%s reason=%s",
+                    _tool_intent.category,
+                    _tool_intent.reason,
+                )
         except SessionNotFoundError as e:
             raise HTTPException(404, str(e))
         except (ValueError, ValidationError):
@@ -775,20 +872,35 @@ def setup_chat_routes(
 
         # Build disabled-tools set from frontend toggles + user privileges
         disabled_tools = set()
-        # Only disable bash/web_search when the caller *explicitly* set them
+        # Only disable bash when the caller *explicitly* set it to a falsy
-        # to a falsy value.  When unset (None), defer to per-user privilege
+        # value. When unset (None), defer to per-user privilege checks below.
-        # checks below — this lets admins with can_use_bash=True use bash
+        # Web search is per-turn opt-in: either the chat pre-search setting
-        # by default without having to send allow_bash in every request.
+        # (`use_web=true`) or agent web toggle (`allow_web_search=true`) must
+        # explicitly enable it.
         if allow_bash is not None and str(allow_bash).lower() != "true":
             disabled_tools.add("bash")
         _explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web")
-        if (
+        if is_web_search_explicitly_denied(allow_web_search) or not _search_enabled:
-            allow_web_search is not None
+            disabled_tools.update(WEB_TOOL_NAMES)
-            and str(allow_web_search).lower() != "true"
+        if _explicit_web_intent:
-            and not _explicit_web_intent
+            # A direct lookup/search request should not drift into personal
-        ):
+            # tools or shell fallbacks. It can only use web_search/web_fetch
-            disabled_tools.add("web_search")
+            # when the request's explicit web setting enabled them.
-            disabled_tools.add("web_fetch")
+            disabled_tools.update({
+                "bash", "python",
+                "search_chats", "manage_skills", "manage_memory",
+                "read_file", "write_file", "edit_file",
+                "create_document", "edit_document", "update_document",
+                "send_email", "reply_to_email",
+                "manage_notes", "manage_calendar", "manage_tasks",
+                "api_call", "builtin_browser",
+            })
+            if _search_enabled:
+                disabled_tools.difference_update(WEB_TOOL_NAMES)
+            else:
+                disabled_tools.update(WEB_TOOL_NAMES)
+        elif _search_enabled:
+            disabled_tools.difference_update(WEB_TOOL_NAMES)
 
         # Nobody/incognito mode: deny tools that would expose the user's
         # persistent memory, past chats, or other identity-linked data.
@@ -839,11 +951,7 @@ def setup_chat_routes(
         from src.settings import get_setting
         _global_disabled = get_setting("disabled_tools", [])
         if _global_disabled and isinstance(_global_disabled, list):
-            explicit_web_allowed = allow_web_search is not None and str(allow_web_search).lower() == "true"
+            disabled_tools.update(_global_disabled)
-            if explicit_web_allowed:
-                disabled_tools.update(t for t in _global_disabled if t not in {"web_search", "web_fetch"})
-            else:
-                disabled_tools.update(_global_disabled)
 
         # Light auto-escalation: the user is in chat mode and just expressed a
         # notes/calendar/email intent. Grant the relevant managers but withhold
@@ -1057,13 +1165,16 @@ def setup_chat_routes(
                     _active_streams.pop(session, None)
                     return
 
-            messages = ctx.messages
+            messages = _ensure_current_request_is_latest_user(ctx.messages, message)
 
             # Auto-compact notification
             if ctx.was_compacted:
                 yield f"data: {json.dumps({'type': 'compacted', 'context_length': ctx.context_length})}\n\n"
+            if ctx.context_trimmed and not ctx.was_compacted:
+                yield f"data: {json.dumps({'type': 'context_trimmed', 'data': {'context_length': ctx.context_length, 'messages_before': ctx.context_messages_before_trim, 'messages_after': ctx.context_messages_after_trim, 'tokens_before': ctx.context_tokens_before_trim, 'tokens_after': ctx.context_tokens_after_trim}})}\n\n"
 
             full_response = ""
+            thinking_response = ""
             last_metrics = None
 
             # Configured fallback chain for the default chat model. Tried in
@@ -1153,7 +1264,9 @@ def setup_chat_routes(
                                     # Forward them so the client can show a thinking
                                     # indicator, but don't fold them into the saved
                                     # reply (mirrors the rewrite path below).
-                                    if not data.get("thinking"):
+                                    if data.get("thinking"):
+                                        thinking_response += data["delta"]
+                                    else:
                                         full_response += data["delta"]
                                         _stream_set(session, partial=full_response)
                                     yield chunk
@@ -1173,6 +1286,12 @@ def setup_chat_routes(
                                     _reported_model = last_metrics.get("model")
                                     last_metrics["requested_model"] = _requested_model
                                     last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
+                                    if ctx.context_trimmed:
+                                        last_metrics["context_trimmed"] = True
+                                        last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
+                                        last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
+                                        last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
+                                        last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
                                     if ctx.context_length and last_metrics.get("input_tokens"):
                                         pct = min(round((last_metrics["input_tokens"] / ctx.context_length) * 100, 1), 100.0)
                                         last_metrics["context_percent"] = pct
@@ -1215,8 +1334,11 @@ def setup_chat_routes(
                                 }
                                 yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
                             if full_response:
+                                _metrics_to_save = dict(last_metrics or {})
+                                if thinking_response.strip() and not _metrics_to_save.get("thinking"):
+                                    _metrics_to_save["thinking"] = thinking_response.strip()
                                 _saved_id = save_assistant_response(
-                                    sess, session_manager, session, full_response, last_metrics,
+                                    sess, session_manager, session, full_response, _metrics_to_save,
                                     character_name=ctx.preset.character_name,
                                     web_sources=web_sources,
                                     rag_sources=ctx.rag_sources,
@@ -1229,7 +1351,7 @@ def setup_chat_routes(
                                     yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
                                 run_post_response_tasks(
                                     sess, session_manager, session, message, full_response,
-                                    last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
+                                    _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
                                     incognito=incognito, compare_mode=compare_mode,
                                     character_name=ctx.preset.character_name,
                                     owner=_user,
@@ -1281,8 +1403,8 @@ def setup_chat_routes(
                     _max_rounds = max(1, min(_max_rounds, 200))
 
                     _forced_tools = None
-                    if allow_web_search is not None and str(allow_web_search).lower() == "true":
+                    if _search_enabled:
-                        _forced_tools = {"web_search", "web_fetch"}
+                        _forced_tools = set(WEB_TOOL_NAMES)
 
                     async for chunk in stream_agent_loop(
                         sess.endpoint_url,
@@ -1315,7 +1437,9 @@ def setup_chat_routes(
                                     # Reasoning tokens arrive flagged thinking:true.
                                     # Forward them for the live indicator, but keep
                                     # them out of the saved reply (same as chat mode).
-                                    if not data.get("thinking"):
+                                    if data.get("thinking"):
+                                        thinking_response += data["delta"]
+                                    else:
                                         full_response += data["delta"]
                                         _stream_set(session, partial=full_response)
                                     yield chunk
@@ -1326,7 +1450,9 @@ def setup_chat_routes(
                                     "tool_start", "tool_output", "agent_step",
                                     "doc_stream_open", "doc_stream_delta",
                                     "doc_update", "doc_suggestions", "ui_control",
-                                    "rounds_exhausted",
+                                    "rounds_exhausted", "budget_exceeded",
+                                    "loop_breaker_triggered",
+                                    "intent_nudge_exhausted",
                                     "ask_user",
                                     "plan_update",
                                 ):
@@ -1353,6 +1479,12 @@ def setup_chat_routes(
                                     _reported_model = last_metrics.get("model")
                                     last_metrics["requested_model"] = last_metrics.get("requested_model") or _requested_model
                                     last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
+                                    if ctx.context_trimmed:
+                                        last_metrics["context_trimmed"] = True
+                                        last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
+                                        last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
+                                        last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
+                                        last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
                                     yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
                             except json.JSONDecodeError:
                                 yield chunk
@@ -1362,8 +1494,11 @@ def setup_chat_routes(
                             _has_tool_events = bool((last_metrics or {}).get("tool_events"))
                             if full_response or _has_tool_events:
                                 _response_to_save = full_response or "Done."
+                                _metrics_to_save = dict(last_metrics or {})
+                                if thinking_response.strip() and not _metrics_to_save.get("thinking"):
+                                    _metrics_to_save["thinking"] = thinking_response.strip()
                                 _saved_id = save_assistant_response(
-                                    sess, session_manager, session, _response_to_save, last_metrics,
+                                    sess, session_manager, session, _response_to_save, _metrics_to_save,
                                     character_name=ctx.preset.character_name,
                                     web_sources=web_sources,
                                     rag_sources=ctx.rag_sources,
@@ -1374,7 +1509,7 @@ def setup_chat_routes(
                                     yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
                                 run_post_response_tasks(
                                     sess, session_manager, session, message, _response_to_save,
-                                    last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
+                                    _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
                                     incognito=incognito, compare_mode=compare_mode,
                                     character_name=ctx.preset.character_name,
                                                             agent_rounds=_agent_rounds,

routes/contacts/__init__.py

@@ -0,0 +1,5 @@
+"""Contacts route domain package (slice 2e, #4082/#4071).
+
+Contains contacts_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/contacts_routes.py re-exports from here.
+"""

routes/contacts/contacts_routes.py

@@ -0,0 +1,916 @@
+"""
+contacts_routes.py
+
+CardDAV contacts integration. Reads from local Radicale, supports
+search and adding new contacts.
+"""
+
+import re
+import logging
+import uuid
+import json
+import csv
+import io
+import os
+import inspect
+import httpx
+from pathlib import Path
+from datetime import datetime
+from urllib.parse import urljoin, urlparse, urlunparse
+
+from core.log_safety import redact_url
+from fastapi import APIRouter, Query, Depends, Response, HTTPException
+from typing import List, Dict, Optional
+
+from core.middleware import require_admin
+from src.url_safety import check_outbound_url
+
+logger = logging.getLogger(__name__)
+
+from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
+DATA_DIR = Path(_DATA_DIR)
+SETTINGS_FILE = Path(_SETTINGS_FILE)
+LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
+
+
+def _load_settings():
+    if SETTINGS_FILE.exists():
+        return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
+    return {}
+
+
+def _save_settings(settings):
+    from core.atomic_io import atomic_write_json
+    atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
+
+
+def _get_carddav_config():
+    import os
+    settings = _load_settings()
+    password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
+    if password and "carddav_password" in settings:
+        from src.secret_storage import decrypt
+        password = decrypt(password)
+    return {
+        "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
+        "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
+        "password": password,
+    }
+
+
+def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
+    cfg = cfg or _get_carddav_config()
+    return bool((cfg.get("url") or "").strip())
+
+
+def _validate_carddav_url(url: str) -> str:
+    cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
+    ok, reason = check_outbound_url(
+        cleaned,
+        block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
+    )
+    if not ok:
+        raise ValueError(f"Rejected CardDAV URL: {reason}")
+    return cleaned
+
+
+def _carddav_base_url(cfg: Dict) -> str:
+    return _validate_carddav_url(cfg.get("url") or "")
+
+
+def _normalize_contact(contact: Dict) -> Dict:
+    emails = []
+    for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
+        e = str(e or "").strip()
+        if e and e not in emails:
+            emails.append(e)
+    phones = []
+    for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
+        p = str(p or "").strip()
+        if p and p not in phones:
+            phones.append(p)
+    name = str(contact.get("name") or "").strip()
+    if not name and emails:
+        name = emails[0].split("@")[0]
+    address = str(contact.get("address") or "").strip()
+    return {
+        "uid": str(contact.get("uid") or uuid.uuid4()),
+        "name": name,
+        "emails": emails,
+        "phones": phones,
+        "address": address,
+    }
+
+
+def _load_local_contacts() -> List[Dict]:
+    try:
+        if not LOCAL_CONTACTS_FILE.exists():
+            return []
+        data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
+        rows = data.get("contacts", data) if isinstance(data, dict) else data
+        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+    except Exception as e:
+        logger.error(f"Failed to load local contacts: {e}")
+        return []
+
+
+def _save_local_contacts(contacts: List[Dict]) -> None:
+    from core.atomic_io import atomic_write_json
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
+    _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
+    _contact_cache["fetched_at"] = datetime.utcnow()
+
+
+# ── vCard parsing ──
+
+def _vunesc(value: str) -> str:
+    """Reverse _vesc() — turn escaped vCard text back into the raw value.
+    Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
+    if not value:
+        return value
+    out = []
+    i = 0
+    while i < len(value):
+        ch = value[i]
+        if ch == "\\" and i + 1 < len(value):
+            nxt = value[i + 1]
+            if nxt in ("n", "N"):
+                out.append("\n")
+            elif nxt in (",", ";", "\\"):
+                out.append(nxt)
+            else:
+                out.append(nxt)
+            i += 2
+        else:
+            out.append(ch)
+            i += 1
+    return "".join(out)
+
+
+def _parse_vcards(text: str) -> List[Dict]:
+    """Parse a stream of vCards into dicts with name, email, phone."""
+    # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
+    # space or tab is a continuation of the previous logical line. Real
+    # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
+    # PHOTO lines, and splitting on raw newlines without unfolding dropped the
+    # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
+    # email/name.
+    text = re.sub(r"\r\n[ \t]", "", text or "")
+    text = re.sub(r"\n[ \t]", "", text)
+    contacts = []
+    for block in re.split(r"BEGIN:VCARD", text):
+        if not block.strip():
+            continue
+        contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
+        for line in block.split("\n"):
+            line = line.strip()
+            # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
+            # that Apple Contacts / iCloud / many CardDAV servers emit by
+            # default — without this the property-name checks below miss those
+            # lines and silently drop the email / phone. The group token only
+            # precedes the property name, so it is safe to strip for matching
+            # and value extraction, and a no-op for non-grouped lines.
+            name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
+            if name_part.startswith("FN:") or name_part.startswith("FN;"):
+                contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
+            elif name_part.startswith("EMAIL"):
+                # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
+                if ":" in name_part:
+                    email_addr = _vunesc(name_part.split(":", 1)[1])
+                    if email_addr and email_addr not in contact["emails"]:
+                        contact["emails"].append(email_addr)
+            elif name_part.startswith("TEL"):
+                if ":" in name_part:
+                    phone = _vunesc(name_part.split(":", 1)[1])
+                    if phone and phone not in contact["phones"]:
+                        contact["phones"].append(phone)
+            elif name_part.startswith("ADR"):
+                # vCard ADR is 7 semicolon-separated components:
+                # post-office-box;extended-address;street;locality;region;postal-code;country.
+                # Recover a human-readable string by joining non-empty
+                # components with ", ".
+                if ":" in name_part:
+                    raw = name_part.split(":", 1)[1]
+                    parts = [_vunesc(p).strip() for p in raw.split(";")]
+                    contact["address"] = ", ".join(p for p in parts if p)
+            elif name_part.startswith("UID:"):
+                contact["uid"] = _vunesc(name_part[4:])
+        if contact["name"] or contact["emails"]:
+            contacts.append(contact)
+    return contacts
+
+
+def _vesc(value: str) -> str:
+    """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
+    semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
+    or any value containing a newline produces a malformed vCard (broken
+    N/FN fields) or could inject arbitrary properties."""
+    return (
+        (value or "")
+        .replace("\\", "\\\\")
+        .replace("\n", "\\n")
+        .replace("\r", "")
+        .replace(",", "\\,")
+        .replace(";", "\\;")
+    )
+
+
+def _build_vcard(name: str, email: str, uid: Optional[str] = None,
+                 emails: Optional[List[str]] = None,
+                 phones: Optional[List[str]] = None,
+                 address: Optional[str] = None) -> str:
+    """Build a vCard. Accepts either a single `email` (legacy callers) or
+    full `emails`/`phones` lists (edit path). The first email is marked
+    PREF=1. All values are RFC-6350-escaped."""
+    if not uid:
+        uid = str(uuid.uuid4())
+    # Normalize email lists — `email` arg is a convenience for single-email
+    # creation; `emails` (if given) is authoritative.
+    email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
+    phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
+    # Try to split name into first/last
+    parts = name.strip().split()
+    if len(parts) >= 2:
+        first = parts[0]
+        last = " ".join(parts[1:])
+    else:
+        first = name
+        last = ""
+    # N field is structured (5 components separated by ';') — escape each
+    # component individually so a comma in the name doesn't split it.
+    n_field = f"{_vesc(last)};{_vesc(first)};;;"
+    lines = [
+        "BEGIN:VCARD",
+        "VERSION:4.0",
+        f"UID:{_vesc(uid)}",
+        f"FN:{_vesc(name)}",
+        f"N:{n_field}",
+    ]
+    for i, em in enumerate(email_list):
+        # First email is the preferred one.
+        lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
+    for ph in phone_list:
+        lines.append(f"TEL:{_vesc(ph)}")
+    # Address: stuff the whole human-readable string into the street
+    # component of ADR. vCard ADR has 7 semicolon-separated components:
+    # post-office-box;extended-address;street;locality;region;postal-code;country.
+    addr = (address or "").strip()
+    if addr:
+        lines.append(f"ADR:;;{_vesc(addr)};;;;")
+    lines.append("END:VCARD")
+    return "\r\n".join(lines) + "\r\n"
+
+
+# ── In-memory cache ──
+
+_contact_cache = {"contacts": [], "fetched_at": None}
+
+
+def _abs_url(href: str) -> str:
+    """Combine a multistatus <href> (an absolute path like
+    /user/contacts/x.vcf) with the configured CardDAV server origin so we
+    get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
+    for the configured origin; a cross-origin href is treated as a path on the
+    configured server so a malicious CardDAV response cannot redirect later
+    writes/deletes to cloud metadata or another host."""
+    cfg = _get_carddav_config()
+    base = _carddav_base_url(cfg)
+    base_p = urlparse(base)
+    joined = urljoin(base.rstrip("/") + "/", href or "")
+    joined_p = urlparse(joined)
+    if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
+        joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
+    return _validate_carddav_url(joined)
+
+
+# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
+# alongside the resource href. Lets us map each contact's UID to the real
+# server resource path (which is NOT always <uid>.vcf for contacts created
+# by other clients).
+_ADDRESSBOOK_QUERY = (
+    '<?xml version="1.0" encoding="utf-8"?>'
+    '<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
+    '<D:prop><D:getetag/><C:address-data/></D:prop>'
+    '<C:filter/>'
+    '</C:addressbook-query>'
+)
+
+
+def _fetch_via_report(cfg, auth):
+    """Try a CardDAV REPORT addressbook-query — returns contacts WITH an
+    `href` field, or None if the server doesn't support it / errors."""
+    from defusedxml import ElementTree as ET
+    try:
+        r = httpx.request(
+            "REPORT", cfg["url"],
+            content=_ADDRESSBOOK_QUERY.encode("utf-8"),
+            headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
+            auth=auth, timeout=10,
+        )
+        if r.status_code not in (207, 200):
+            return None
+        root = ET.fromstring(r.text)
+        ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
+        out = []
+        for resp in root.findall("D:response", ns):
+            href_el = resp.find("D:href", ns)
+            data_el = resp.find(".//C:address-data", ns)
+            if href_el is None or data_el is None or not (data_el.text or "").strip():
+                continue
+            parsed = _parse_vcards(data_el.text)
+            if not parsed:
+                continue
+            c = parsed[0]
+            c["href"] = href_el.text.strip()
+            out.append(c)
+        # If the REPORT parsed to ZERO contacts, don't trust it — some
+        # CardDAV servers treat an empty <filter/> as "match nothing" and
+        # return a valid-but-empty 207. Return None so the caller falls
+        # back to the plain GET (which lists everything). A genuinely empty
+        # address book just costs one extra GET that also returns nothing.
+        if not out:
+            return None
+        return out
+    except Exception as e:
+        logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
+        return None
+
+
+def _fetch_contacts(force=False):
+    """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
+    if not force and _contact_cache["fetched_at"]:
+        age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
+        if age < 60:
+            return _contact_cache["contacts"]
+
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        _contact_cache["contacts"] = contacts
+        _contact_cache["fetched_at"] = datetime.utcnow()
+        return contacts
+
+    try:
+        cfg["url"] = _carddav_base_url(cfg)
+        auth = None
+        if cfg["username"]:
+            auth = (cfg["username"], cfg["password"])
+        # Preferred path: REPORT gives us hrefs for reliable edit/delete.
+        contacts = _fetch_via_report(cfg, auth)
+        if contacts is None:
+            # Fallback: plain GET, concatenated vCards, no hrefs.
+            r = httpx.get(cfg["url"], auth=auth, timeout=10)
+            if r.status_code != 200:
+                logger.warning(f"CardDAV returned {r.status_code}")
+                return _contact_cache["contacts"]
+            contacts = _parse_vcards(r.text)
+        _contact_cache["contacts"] = contacts
+        _contact_cache["fetched_at"] = datetime.utcnow()
+        return contacts
+    except Exception as e:
+        logger.error(f"Failed to fetch contacts: {e}")
+        return _contact_cache["contacts"]
+
+
+def _resolve_resource_url(uid: str) -> str:
+    """Map a contact UID to its real CardDAV resource URL. Uses the href
+    captured during fetch when available (handles contacts whose filename
+    != UID); falls back to the <uid>.vcf guess for app-created contacts or
+    when no href is known."""
+    def _lookup():
+        for c in _contact_cache.get("contacts", []):
+            if c.get("uid") == uid and c.get("href"):
+                return _abs_url(c["href"])
+        return None
+    found = _lookup()
+    if found:
+        return found
+    # Not in cache (or no href) — refresh once and retry before guessing.
+    try:
+        _fetch_contacts(force=True)
+    except Exception:
+        pass
+    return _lookup() or _vcard_url(uid)
+
+
+def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
+    """Add a new contact via CardDAV or local contacts."""
+    email = (email or "").strip()
+    phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        email_l = email.lower()
+        for c in contacts:
+            if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
+                return True
+            if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
+                return True
+        contacts.append(_normalize_contact({
+            "name": name,
+            "emails": [email] if email else [],
+            "phones": phone_list,
+            "address": address,
+        }))
+        _save_local_contacts(contacts)
+        return True
+
+    contact_uid = str(uuid.uuid4())
+    vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
+    try:
+        url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
+        auth = None
+        if cfg["username"]:
+            auth = (cfg["username"], cfg["password"])
+        r = httpx.put(
+            url,
+            data=vcard.encode("utf-8"),
+            headers={"Content-Type": "text/vcard; charset=utf-8"},
+            auth=auth,
+            timeout=10,
+        )
+        if r.status_code in (200, 201, 204):
+            # Invalidate cache
+            _contact_cache["fetched_at"] = None
+            return True
+        logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to create contact: {e}")
+        return False
+
+
+def _vcard_url(uid: str) -> str:
+    """The CardDAV resource URL for a given contact UID. The uid is URL-
+    encoded so a value containing '/', '..' or other path chars can't
+    escape the collection and target an arbitrary CardDAV resource."""
+    from urllib.parse import quote
+    cfg = _get_carddav_config()
+    return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
+
+
+def _import_vcards(text: str) -> Dict:
+    """Import a (possibly multi-card) .vcf blob. Each card is PUT to the
+    CardDAV server PRESERVING its full original content (ADR/ORG/photo/
+    etc.) — we don't rebuild it, just ensure it has VERSION + UID and
+    normalize line endings. Returns {imported, failed, total}."""
+    from urllib.parse import quote
+    cfg = _get_carddav_config()
+    if not cfg.get("url"):
+        parsed = _parse_vcards(text)
+        contacts = _load_local_contacts()
+        existing = {
+            e.lower()
+            for c in contacts
+            for e in (c.get("emails") or [])
+            if e
+        }
+        imported = 0
+        for c in parsed:
+            emails = [e for e in (c.get("emails") or []) if e]
+            if emails and any(e.lower() in existing for e in emails):
+                continue
+            contacts.append(_normalize_contact(c))
+            for e in emails:
+                existing.add(e.lower())
+            imported += 1
+        if imported:
+            _save_local_contacts(contacts)
+        return {"imported": imported, "failed": 0, "total": len(parsed)}
+    try:
+        base_url = _carddav_base_url(cfg)
+    except ValueError as e:
+        logger.warning("CardDAV import URL rejected: %s", e)
+        return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
+    auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
+    # Split into individual cards. re.split drops the BEGIN line, so we
+    # re-add it. Normalize CRLF.
+    raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
+    blocks = []
+    for chunk in raw.split("BEGIN:VCARD"):
+        chunk = chunk.strip()
+        if not chunk:
+            continue
+        # Trim anything after END:VCARD (defensive).
+        end = chunk.upper().find("END:VCARD")
+        body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
+        blocks.append("BEGIN:VCARD\n" + body)
+    imported = 0
+    failed = 0
+    for block in blocks:
+        # Extract or assign a UID.
+        m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
+        uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
+        if not m:
+            # Inject a UID right after the VERSION line (or after BEGIN).
+            if re.search(r"^VERSION:", block, re.MULTILINE):
+                block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
+            else:
+                block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
+        elif not re.search(r"^VERSION:", block, re.MULTILINE):
+            block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
+        vcard = block.replace("\n", "\r\n") + "\r\n"
+        url = base_url + "/" + quote(uid, safe="") + ".vcf"
+        try:
+            r = httpx.put(
+                url, data=vcard.encode("utf-8"),
+                headers={"Content-Type": "text/vcard; charset=utf-8"},
+                auth=auth, timeout=15,
+            )
+            if r.status_code in (200, 201, 204):
+                imported += 1
+            else:
+                failed += 1
+                logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
+        except Exception as e:
+            failed += 1
+            logger.error(f"Import PUT {uid} failed: {e}")
+    if imported:
+        _contact_cache["fetched_at"] = None
+    return {"imported": imported, "failed": failed, "total": len(blocks)}
+
+
+def _import_csv_contacts(text: str) -> Dict:
+    """Import contacts from CSV. Supports common headers:
+    name/full_name/display_name, email/email_address/e-mail, phone/tel.
+    Falls back to first columns as name,email,phone when no headers exist."""
+    raw = (text or "").strip()
+    if not raw:
+        return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
+
+    try:
+        sample = raw[:2048]
+        dialect = csv.Sniffer().sniff(sample)
+    except Exception:
+        dialect = csv.excel
+
+    stream = io.StringIO(raw)
+    try:
+        has_header = csv.Sniffer().has_header(raw[:2048])
+    except Exception:
+        has_header = True
+
+    rows = []
+    if has_header:
+        reader = csv.DictReader(stream, dialect=dialect)
+        for row in reader:
+            lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
+            name = (
+                lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
+                or lowered.get("display name") or lowered.get("display_name")
+                or lowered.get("fn") or ""
+            )
+            email = (
+                lowered.get("email") or lowered.get("email address")
+                or lowered.get("email_address") or lowered.get("e-mail")
+                or lowered.get("mail") or ""
+            )
+            phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
+            rows.append((name, email, phone))
+    else:
+        stream.seek(0)
+        reader = csv.reader(stream, dialect=dialect)
+        for row in reader:
+            cols = [(c or "").strip() for c in row]
+            if not any(cols):
+                continue
+            rows.append((
+                cols[0] if len(cols) > 0 else "",
+                cols[1] if len(cols) > 1 else "",
+                cols[2] if len(cols) > 2 else "",
+            ))
+
+    imported = 0
+    failed = 0
+    total = 0
+    existing_emails = {
+        e.lower()
+        for c in _fetch_contacts()
+        for e in (c.get("emails") or [])
+        if e
+    }
+    for name, email, phone in rows:
+        email = (email or "").strip()
+        name = (name or "").strip() or (email.split("@")[0] if email else "")
+        if not email:
+            continue
+        total += 1
+        if email.lower() in existing_emails:
+            continue
+        ok = _create_contact(name, email)
+        if ok:
+            imported += 1
+            existing_emails.add(email.lower())
+            # If the CSV had a phone number, rewrite the just-created row
+            # through the richer update path so phone lands in CardDAV too.
+            if phone:
+                try:
+                    contacts = _fetch_contacts(force=True)
+                    created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
+                    if created and created.get("uid"):
+                        _update_contact(created["uid"], name, [email], [phone])
+                except Exception:
+                    pass
+        else:
+            failed += 1
+
+    if imported:
+        _contact_cache["fetched_at"] = None
+    return {"imported": imported, "failed": failed, "total": total}
+
+
+def _contacts_to_vcf(contacts: List[Dict]) -> str:
+    return "".join(
+        _build_vcard(
+            c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
+            "",
+            uid=c.get("uid") or str(uuid.uuid4()),
+            emails=c.get("emails") or [],
+            phones=c.get("phones") or [],
+        )
+        for c in contacts
+    )
+
+
+def _contacts_to_csv(contacts: List[Dict]) -> str:
+    out = io.StringIO()
+    writer = csv.writer(out)
+    writer.writerow(["name", "email", "phone"])
+    for c in contacts:
+        emails = c.get("emails") or [""]
+        phones = c.get("phones") or [""]
+        max_len = max(len(emails), len(phones), 1)
+        for i in range(max_len):
+            writer.writerow([
+                c.get("name") or "",
+                emails[i] if i < len(emails) else "",
+                phones[i] if i < len(phones) else "",
+            ])
+    return out.getvalue()
+
+
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
+    """Rewrite an existing contact via CardDAV or local contacts."""
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        found = False
+        out = []
+        for c in contacts:
+            if c.get("uid") == uid:
+                # Preserve existing address when caller passes "" (only
+                # updating name/emails/phones, not touching address).
+                addr = address if address else c.get("address", "")
+                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
+                found = True
+            else:
+                out.append(c)
+        if not found:
+            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
+        _save_local_contacts(out)
+        return True
+
+    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
+    # Use the real resource href (handles externally-created contacts whose
+    # filename != UID); falls back to the <uid>.vcf guess.
+    try:
+        url = _resolve_resource_url(uid)
+        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
+        r = httpx.put(
+            url,
+            data=vcard.encode("utf-8"),
+            headers={"Content-Type": "text/vcard; charset=utf-8"},
+            auth=auth,
+            timeout=10,
+        )
+        if r.status_code in (200, 201, 204):
+            _contact_cache["fetched_at"] = None
+            return True
+        logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to update contact: {e}")
+        return False
+
+
+def _delete_contact(uid: str) -> bool:
+    """Delete a contact via CardDAV or local contacts."""
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        remaining = [c for c in contacts if c.get("uid") != uid]
+        _save_local_contacts(remaining)
+        return True
+
+    try:
+        url = _resolve_resource_url(uid)
+        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
+        r = httpx.delete(url, auth=auth, timeout=10)
+        if r.status_code in (200, 204, 404):
+            # Invalidate cache so the next fetch sees the server truth.
+            _contact_cache["fetched_at"] = None
+            # Verify: force a fresh fetch and check the UID is actually gone.
+            # A 404 on the guessed URL ({uid}.vcf) can mean the contact
+            # lives at a different resource URL — the DELETE missed it but
+            # we'd silently report success. This check catches that.
+            fresh = _fetch_contacts(force=True)
+            still_there = any(c.get("uid") == uid for c in fresh)
+            if still_there:
+                logger.warning(
+                    f"CardDAV DELETE reported success for {uid} "
+                    f"but UID still present after re-fetch — "
+                    f"resource URL may differ from {redact_url(url)}"
+                )
+                return False
+            if r.status_code == 404:
+                logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
+            return True
+        logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to delete contact: {e}")
+        return False
+
+
+# ── Routes ──
+
+def setup_contacts_routes():
+    router = APIRouter(prefix="/api/contacts", tags=["contacts"])
+
+    @router.get("/list")
+    async def list_contacts(_admin: str = Depends(require_admin)):
+        """List all contacts."""
+        contacts = _fetch_contacts()
+        return {"contacts": contacts, "count": len(contacts)}
+
+    @router.get("/search")
+    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
+        """Search contacts by name or email. Returns up to 10 matches."""
+        contacts = _fetch_contacts()
+        if not q:
+            return {"results": []}
+        q_lower = q.lower()
+        results = []
+        for c in contacts:
+            if q_lower in c["name"].lower():
+                results.append(c)
+                continue
+            for em in c["emails"]:
+                if q_lower in em.lower():
+                    results.append(c)
+                    break
+        return {"results": results[:10]}
+
+    @router.post("/add")
+    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
+        """Add a new contact."""
+        name = (data.get("name") or "").strip()
+        email = (data.get("email") or "").strip()
+        phone = (data.get("phone") or "").strip()
+        phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
+        if phone and phone not in phones:
+            phones.insert(0, phone)
+        address = (data.get("address") or "").strip()
+        if not name and email:
+            name = email.split("@")[0]
+        if not name and not email and not phones and not address:
+            return {"success": False, "error": "Name, email, phone, or address required"}
+        if not name:
+            name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
+        contacts = _fetch_contacts()
+        for c in contacts:
+            if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
+                return {"success": True, "message": "Already exists", "contact": c}
+            if phones and any(p in (c.get("phones") or []) for p in phones):
+                return {"success": True, "message": "Already exists", "contact": c}
+        create_params = inspect.signature(_create_contact).parameters
+        if "phones" in create_params:
+            ok = _create_contact(name, email, address, phones=phones)
+        elif len(create_params) >= 3:
+            ok = _create_contact(name, email, address)
+        else:
+            ok = _create_contact(name, email)
+        # If a phone was provided, do an immediate update to thread it
+        # through (the simple _create_contact signature only takes name +
+        # email + address; phones happen via update).
+        if ok and phones and "phones" not in create_params:
+            try:
+                fresh = _fetch_contacts(force=True)
+                created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
+                if created:
+                    _update_contact(
+                        created["uid"], name,
+                        created.get("emails", []),
+                        phones,
+                        address,
+                    )
+            except Exception:
+                pass
+        return {"success": ok}
+
+    @router.post("/import")
+    async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
+        """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
+        # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
+        # in the JSON body) would otherwise reach .strip() and 500 with an
+        # AttributeError instead of degrading to a clean "no data" response.
+        text = str(data.get("vcf") or data.get("text") or "")
+        csv_text = str(data.get("csv") or "")
+        if text.strip():
+            if "BEGIN:VCARD" not in text.upper():
+                return {"success": False, "error": "No vCard data found"}
+            result = _import_vcards(text)
+        elif csv_text.strip():
+            result = _import_csv_contacts(csv_text)
+        else:
+            return {"success": False, "error": "No contact data found"}
+        result["success"] = result.get("imported", 0) > 0
+        return result
+
+    @router.get("/export")
+    async def export_contacts(
+        format: str = Query("vcf", pattern="^(vcf|csv)$"),
+        _admin: str = Depends(require_admin),
+    ):
+        """Export all contacts as vCard or CSV."""
+        contacts = _fetch_contacts(force=True)
+        if format == "csv":
+            content = _contacts_to_csv(contacts)
+            media_type = "text/csv; charset=utf-8"
+            filename = "odysseus-contacts.csv"
+        else:
+            content = _contacts_to_vcf(contacts)
+            media_type = "text/vcard; charset=utf-8"
+            filename = "odysseus-contacts.vcf"
+        return Response(
+            content=content,
+            media_type=media_type,
+            headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+        )
+
+    @router.get("/config")
+    async def get_config(_admin: str = Depends(require_admin)):
+        cfg = _get_carddav_config()
+        # Mask password
+        if cfg["password"]:
+            cfg["password"] = "***"
+        return cfg
+
+    @router.put("/config")
+    async def update_config(data: dict, _admin: str = Depends(require_admin)):
+        settings = _load_settings()
+        for key in ("carddav_url", "carddav_username", "carddav_password"):
+            if key in data:
+                if key == "carddav_url" and str(data[key] or "").strip():
+                    try:
+                        settings[key] = _validate_carddav_url(data[key])
+                    except ValueError as e:
+                        raise HTTPException(400, str(e))
+                else:
+                    value = data[key]
+                    if key == "carddav_password" and value:
+                        from src.secret_storage import encrypt
+                        value = encrypt(value)
+                    settings[key] = value
+        _save_settings(settings)
+        # Force re-fetch
+        _contact_cache["fetched_at"] = None
+        return {"success": True}
+
+    @router.delete("/clear")
+    async def clear_contacts(_admin: str = Depends(require_admin)):
+        """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
+        _save_local_contacts([])
+        return {"success": True}
+
+    # NOTE: the /{uid} routes are declared LAST so the literal paths above
+    # (/list, /search, /add, /config) win — otherwise PUT /config would
+    # match PUT /{uid} with uid="config".
+    @router.put("/{uid}")
+    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
+        """Edit an existing contact — name / emails / phones / address."""
+        name = (data.get("name") or "").strip()
+        emails = data.get("emails")
+        phones = data.get("phones")
+        if emails is None and data.get("email"):
+            emails = [data["email"]]
+        emails = [e.strip() for e in (emails or []) if e and e.strip()]
+        phones = [p.strip() for p in (phones or []) if p and p.strip()]
+        address = (data.get("address") or "").strip()
+        if not name and not emails and not address:
+            return {"success": False, "error": "Name, email, or address required"}
+        if not name and emails:
+            name = emails[0].split("@")[0]
+        ok = _update_contact(uid, name, emails, phones, address)
+        return {"success": ok}
+
+    @router.delete("/{uid}")
+    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
+        """Delete a contact by UID."""
+        if not uid:
+            return {"success": False, "error": "UID required"}
+        ok = _delete_contact(uid)
+        return {"success": ok}
+
+    return router

routes/contacts_routes.py

@@ -1,900 +1,13 @@
-"""
+"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py.
-contacts_routes.py
 
-CardDAV contacts integration. Reads from local Radicale, supports
+This module is replaced in ``sys.modules`` by the canonical module object so
-search and adding new contacts.
+that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``,
+``importlib.import_module("routes.contacts_routes")``, and string-targeted
+monkeypatches all operate on the same object the application actually uses.
 """
 
-import re
+import sys as _sys
-import logging
-import uuid
-import json
-import csv
-import io
-import os
-import inspect
-import httpx
-from pathlib import Path
-from datetime import datetime
-from urllib.parse import urljoin, urlparse, urlunparse
 
-from core.log_safety import redact_url
+from routes.contacts import contacts_routes as _canonical  # noqa: F401
-from fastapi import APIRouter, Query, Depends, Response, HTTPException
-from typing import List, Dict, Optional
 
-from core.middleware import require_admin
+_sys.modules[__name__] = _canonical
-from src.url_safety import check_outbound_url
-
-logger = logging.getLogger(__name__)
-
-from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
-DATA_DIR = Path(_DATA_DIR)
-SETTINGS_FILE = Path(_SETTINGS_FILE)
-LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
-
-
-def _load_settings():
-    if SETTINGS_FILE.exists():
-        return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
-    return {}
-
-
-def _save_settings(settings):
-    from core.atomic_io import atomic_write_json
-    atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
-
-
-def _get_carddav_config():
-    import os
-    settings = _load_settings()
-    password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
-    if password and "carddav_password" in settings:
-        from src.secret_storage import decrypt
-        password = decrypt(password)
-    return {
-        "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
-        "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
-        "password": password,
-    }
-
-
-def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
-    cfg = cfg or _get_carddav_config()
-    return bool((cfg.get("url") or "").strip())
-
-
-def _validate_carddav_url(url: str) -> str:
-    cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
-    ok, reason = check_outbound_url(
-        cleaned,
-        block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
-    )
-    if not ok:
-        raise ValueError(f"Rejected CardDAV URL: {reason}")
-    return cleaned
-
-
-def _carddav_base_url(cfg: Dict) -> str:
-    return _validate_carddav_url(cfg.get("url") or "")
-
-
-def _normalize_contact(contact: Dict) -> Dict:
-    emails = []
-    for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
-        e = str(e or "").strip()
-        if e and e not in emails:
-            emails.append(e)
-    phones = []
-    for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
-        p = str(p or "").strip()
-        if p and p not in phones:
-            phones.append(p)
-    name = str(contact.get("name") or "").strip()
-    if not name and emails:
-        name = emails[0].split("@")[0]
-    address = str(contact.get("address") or "").strip()
-    return {
-        "uid": str(contact.get("uid") or uuid.uuid4()),
-        "name": name,
-        "emails": emails,
-        "phones": phones,
-        "address": address,
-    }
-
-
-def _load_local_contacts() -> List[Dict]:
-    try:
-        if not LOCAL_CONTACTS_FILE.exists():
-            return []
-        data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
-        rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
-    except Exception as e:
-        logger.error(f"Failed to load local contacts: {e}")
-        return []
-
-
-def _save_local_contacts(contacts: List[Dict]) -> None:
-    from core.atomic_io import atomic_write_json
-    DATA_DIR.mkdir(parents=True, exist_ok=True)
-    atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
-    _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
-    _contact_cache["fetched_at"] = datetime.utcnow()
-
-
-# ── vCard parsing ──
-
-def _vunesc(value: str) -> str:
-    """Reverse _vesc() — turn escaped vCard text back into the raw value.
-    Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
-    if not value:
-        return value
-    out = []
-    i = 0
-    while i < len(value):
-        ch = value[i]
-        if ch == "\\" and i + 1 < len(value):
-            nxt = value[i + 1]
-            if nxt in ("n", "N"):
-                out.append("\n")
-            elif nxt in (",", ";", "\\"):
-                out.append(nxt)
-            else:
-                out.append(nxt)
-            i += 2
-        else:
-            out.append(ch)
-            i += 1
-    return "".join(out)
-
-
-def _parse_vcards(text: str) -> List[Dict]:
-    """Parse a stream of vCards into dicts with name, email, phone."""
-    # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
-    # space or tab is a continuation of the previous logical line. Real
-    # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
-    # PHOTO lines, and splitting on raw newlines without unfolding dropped the
-    # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
-    # email/name.
-    text = re.sub(r"\r\n[ \t]", "", text or "")
-    text = re.sub(r"\n[ \t]", "", text)
-    contacts = []
-    for block in re.split(r"BEGIN:VCARD", text):
-        if not block.strip():
-            continue
-        contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
-        for line in block.split("\n"):
-            line = line.strip()
-            # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
-            # that Apple Contacts / iCloud / many CardDAV servers emit by
-            # default — without this the property-name checks below miss those
-            # lines and silently drop the email / phone. The group token only
-            # precedes the property name, so it is safe to strip for matching
-            # and value extraction, and a no-op for non-grouped lines.
-            name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
-            if name_part.startswith("FN:") or name_part.startswith("FN;"):
-                contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
-            elif name_part.startswith("EMAIL"):
-                # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
-                if ":" in name_part:
-                    email_addr = _vunesc(name_part.split(":", 1)[1])
-                    if email_addr and email_addr not in contact["emails"]:
-                        contact["emails"].append(email_addr)
-            elif name_part.startswith("TEL"):
-                if ":" in name_part:
-                    phone = _vunesc(name_part.split(":", 1)[1])
-                    if phone and phone not in contact["phones"]:
-                        contact["phones"].append(phone)
-            elif name_part.startswith("ADR"):
-                # vCard ADR is 7 semicolon-separated components:
-                # post-office-box;extended-address;street;locality;region;postal-code;country.
-                # Recover a human-readable string by joining non-empty
-                # components with ", ".
-                if ":" in name_part:
-                    raw = name_part.split(":", 1)[1]
-                    parts = [_vunesc(p).strip() for p in raw.split(";")]
-                    contact["address"] = ", ".join(p for p in parts if p)
-            elif name_part.startswith("UID:"):
-                contact["uid"] = _vunesc(name_part[4:])
-        if contact["name"] or contact["emails"]:
-            contacts.append(contact)
-    return contacts
-
-
-def _vesc(value: str) -> str:
-    """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
-    semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
-    or any value containing a newline produces a malformed vCard (broken
-    N/FN fields) or could inject arbitrary properties."""
-    return (
-        (value or "")
-        .replace("\\", "\\\\")
-        .replace("\n", "\\n")
-        .replace("\r", "")
-        .replace(",", "\\,")
-        .replace(";", "\\;")
-    )
-
-
-def _build_vcard(name: str, email: str, uid: Optional[str] = None,
-                 emails: Optional[List[str]] = None,
-                 phones: Optional[List[str]] = None,
-                 address: Optional[str] = None) -> str:
-    """Build a vCard. Accepts either a single `email` (legacy callers) or
-    full `emails`/`phones` lists (edit path). The first email is marked
-    PREF=1. All values are RFC-6350-escaped."""
-    if not uid:
-        uid = str(uuid.uuid4())
-    # Normalize email lists — `email` arg is a convenience for single-email
-    # creation; `emails` (if given) is authoritative.
-    email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
-    phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
-    # Try to split name into first/last
-    parts = name.strip().split()
-    if len(parts) >= 2:
-        first = parts[0]
-        last = " ".join(parts[1:])
-    else:
-        first = name
-        last = ""
-    # N field is structured (5 components separated by ';') — escape each
-    # component individually so a comma in the name doesn't split it.
-    n_field = f"{_vesc(last)};{_vesc(first)};;;"
-    lines = [
-        "BEGIN:VCARD",
-        "VERSION:4.0",
-        f"UID:{_vesc(uid)}",
-        f"FN:{_vesc(name)}",
-        f"N:{n_field}",
-    ]
-    for i, em in enumerate(email_list):
-        # First email is the preferred one.
-        lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
-    for ph in phone_list:
-        lines.append(f"TEL:{_vesc(ph)}")
-    # Address: stuff the whole human-readable string into the street
-    # component of ADR. vCard ADR has 7 semicolon-separated components:
-    # post-office-box;extended-address;street;locality;region;postal-code;country.
-    addr = (address or "").strip()
-    if addr:
-        lines.append(f"ADR:;;{_vesc(addr)};;;;")
-    lines.append("END:VCARD")
-    return "\r\n".join(lines) + "\r\n"
-
-
-# ── In-memory cache ──
-
-_contact_cache = {"contacts": [], "fetched_at": None}
-
-
-def _abs_url(href: str) -> str:
-    """Combine a multistatus <href> (an absolute path like
-    /user/contacts/x.vcf) with the configured CardDAV server origin so we
-    get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
-    for the configured origin; a cross-origin href is treated as a path on the
-    configured server so a malicious CardDAV response cannot redirect later
-    writes/deletes to cloud metadata or another host."""
-    cfg = _get_carddav_config()
-    base = _carddav_base_url(cfg)
-    base_p = urlparse(base)
-    joined = urljoin(base.rstrip("/") + "/", href or "")
-    joined_p = urlparse(joined)
-    if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
-        joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
-    return _validate_carddav_url(joined)
-
-
-# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
-# alongside the resource href. Lets us map each contact's UID to the real
-# server resource path (which is NOT always <uid>.vcf for contacts created
-# by other clients).
-_ADDRESSBOOK_QUERY = (
-    '<?xml version="1.0" encoding="utf-8"?>'
-    '<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
-    '<D:prop><D:getetag/><C:address-data/></D:prop>'
-    '<C:filter/>'
-    '</C:addressbook-query>'
-)
-
-
-def _fetch_via_report(cfg, auth):
-    """Try a CardDAV REPORT addressbook-query — returns contacts WITH an
-    `href` field, or None if the server doesn't support it / errors."""
-    from defusedxml import ElementTree as ET
-    try:
-        r = httpx.request(
-            "REPORT", cfg["url"],
-            content=_ADDRESSBOOK_QUERY.encode("utf-8"),
-            headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
-        )
-        if r.status_code not in (207, 200):
-            return None
-        root = ET.fromstring(r.text)
-        ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
-        out = []
-        for resp in root.findall("D:response", ns):
-            href_el = resp.find("D:href", ns)
-            data_el = resp.find(".//C:address-data", ns)
-            if href_el is None or data_el is None or not (data_el.text or "").strip():
-                continue
-            parsed = _parse_vcards(data_el.text)
-            if not parsed:
-                continue
-            c = parsed[0]
-            c["href"] = href_el.text.strip()
-            out.append(c)
-        # If the REPORT parsed to ZERO contacts, don't trust it — some
-        # CardDAV servers treat an empty <filter/> as "match nothing" and
-        # return a valid-but-empty 207. Return None so the caller falls
-        # back to the plain GET (which lists everything). A genuinely empty
-        # address book just costs one extra GET that also returns nothing.
-        if not out:
-            return None
-        return out
-    except Exception as e:
-        logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
-        return None
-
-
-def _fetch_contacts(force=False):
-    """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
-        age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
-            return _contact_cache["contacts"]
-
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
-        return contacts
-
-    try:
-        cfg["url"] = _carddav_base_url(cfg)
-        auth = None
-        if cfg["username"]:
-            auth = (cfg["username"], cfg["password"])
-        # Preferred path: REPORT gives us hrefs for reliable edit/delete.
-        contacts = _fetch_via_report(cfg, auth)
-        if contacts is None:
-            # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
-            if r.status_code != 200:
-                logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
-            contacts = _parse_vcards(r.text)
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
-        return contacts
-    except Exception as e:
-        logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
-
-
-def _resolve_resource_url(uid: str) -> str:
-    """Map a contact UID to its real CardDAV resource URL. Uses the href
-    captured during fetch when available (handles contacts whose filename
-    != UID); falls back to the <uid>.vcf guess for app-created contacts or
-    when no href is known."""
-    def _lookup():
-        for c in _contact_cache.get("contacts", []):
-            if c.get("uid") == uid and c.get("href"):
-                return _abs_url(c["href"])
-        return None
-    found = _lookup()
-    if found:
-        return found
-    # Not in cache (or no href) — refresh once and retry before guessing.
-    try:
-        _fetch_contacts(force=True)
-    except Exception:
-        pass
-    return _lookup() or _vcard_url(uid)
-
-
-def _create_contact(name: str, email: str, address: str = "") -> bool:
-    """Add a new contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        email_l = (email or "").strip().lower()
-        for c in contacts:
-            if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
-                return True
-        contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address}))
-        _save_local_contacts(contacts)
-        return True
-
-    contact_uid = str(uuid.uuid4())
-    vcard = _build_vcard(name, email, contact_uid, address=address)
-    try:
-        url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
-        auth = None
-        if cfg["username"]:
-            auth = (cfg["username"], cfg["password"])
-        r = httpx.put(
-            url,
-            data=vcard.encode("utf-8"),
-            headers={"Content-Type": "text/vcard; charset=utf-8"},
-            auth=auth,
-            timeout=10,
-        )
-        if r.status_code in (200, 201, 204):
-            # Invalidate cache
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to create contact: {e}")
-        return False
-
-
-def _vcard_url(uid: str) -> str:
-    """The CardDAV resource URL for a given contact UID. The uid is URL-
-    encoded so a value containing '/', '..' or other path chars can't
-    escape the collection and target an arbitrary CardDAV resource."""
-    from urllib.parse import quote
-    cfg = _get_carddav_config()
-    return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
-
-
-def _import_vcards(text: str) -> Dict:
-    """Import a (possibly multi-card) .vcf blob. Each card is PUT to the
-    CardDAV server PRESERVING its full original content (ADR/ORG/photo/
-    etc.) — we don't rebuild it, just ensure it has VERSION + UID and
-    normalize line endings. Returns {imported, failed, total}."""
-    from urllib.parse import quote
-    cfg = _get_carddav_config()
-    if not cfg.get("url"):
-        parsed = _parse_vcards(text)
-        contacts = _load_local_contacts()
-        existing = {
-            e.lower()
-            for c in contacts
-            for e in (c.get("emails") or [])
-            if e
-        }
-        imported = 0
-        for c in parsed:
-            emails = [e for e in (c.get("emails") or []) if e]
-            if emails and any(e.lower() in existing for e in emails):
-                continue
-            contacts.append(_normalize_contact(c))
-            for e in emails:
-                existing.add(e.lower())
-            imported += 1
-        if imported:
-            _save_local_contacts(contacts)
-        return {"imported": imported, "failed": 0, "total": len(parsed)}
-    try:
-        base_url = _carddav_base_url(cfg)
-    except ValueError as e:
-        logger.warning("CardDAV import URL rejected: %s", e)
-        return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
-    auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-    # Split into individual cards. re.split drops the BEGIN line, so we
-    # re-add it. Normalize CRLF.
-    raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
-    blocks = []
-    for chunk in raw.split("BEGIN:VCARD"):
-        chunk = chunk.strip()
-        if not chunk:
-            continue
-        # Trim anything after END:VCARD (defensive).
-        end = chunk.upper().find("END:VCARD")
-        body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
-        blocks.append("BEGIN:VCARD\n" + body)
-    imported = 0
-    failed = 0
-    for block in blocks:
-        # Extract or assign a UID.
-        m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
-        uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
-        if not m:
-            # Inject a UID right after the VERSION line (or after BEGIN).
-            if re.search(r"^VERSION:", block, re.MULTILINE):
-                block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
-            else:
-                block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
-        elif not re.search(r"^VERSION:", block, re.MULTILINE):
-            block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
-        vcard = block.replace("\n", "\r\n") + "\r\n"
-        url = base_url + "/" + quote(uid, safe="") + ".vcf"
-        try:
-            r = httpx.put(
-                url, data=vcard.encode("utf-8"),
-                headers={"Content-Type": "text/vcard; charset=utf-8"},
-                auth=auth, timeout=15,
-            )
-            if r.status_code in (200, 201, 204):
-                imported += 1
-            else:
-                failed += 1
-                logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
-        except Exception as e:
-            failed += 1
-            logger.error(f"Import PUT {uid} failed: {e}")
-    if imported:
-        _contact_cache["fetched_at"] = None
-    return {"imported": imported, "failed": failed, "total": len(blocks)}
-
-
-def _import_csv_contacts(text: str) -> Dict:
-    """Import contacts from CSV. Supports common headers:
-    name/full_name/display_name, email/email_address/e-mail, phone/tel.
-    Falls back to first columns as name,email,phone when no headers exist."""
-    raw = (text or "").strip()
-    if not raw:
-        return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
-
-    try:
-        sample = raw[:2048]
-        dialect = csv.Sniffer().sniff(sample)
-    except Exception:
-        dialect = csv.excel
-
-    stream = io.StringIO(raw)
-    try:
-        has_header = csv.Sniffer().has_header(raw[:2048])
-    except Exception:
-        has_header = True
-
-    rows = []
-    if has_header:
-        reader = csv.DictReader(stream, dialect=dialect)
-        for row in reader:
-            lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
-            name = (
-                lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
-                or lowered.get("display name") or lowered.get("display_name")
-                or lowered.get("fn") or ""
-            )
-            email = (
-                lowered.get("email") or lowered.get("email address")
-                or lowered.get("email_address") or lowered.get("e-mail")
-                or lowered.get("mail") or ""
-            )
-            phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
-            rows.append((name, email, phone))
-    else:
-        stream.seek(0)
-        reader = csv.reader(stream, dialect=dialect)
-        for row in reader:
-            cols = [(c or "").strip() for c in row]
-            if not any(cols):
-                continue
-            rows.append((
-                cols[0] if len(cols) > 0 else "",
-                cols[1] if len(cols) > 1 else "",
-                cols[2] if len(cols) > 2 else "",
-            ))
-
-    imported = 0
-    failed = 0
-    total = 0
-    existing_emails = {
-        e.lower()
-        for c in _fetch_contacts()
-        for e in (c.get("emails") or [])
-        if e
-    }
-    for name, email, phone in rows:
-        email = (email or "").strip()
-        name = (name or "").strip() or (email.split("@")[0] if email else "")
-        if not email:
-            continue
-        total += 1
-        if email.lower() in existing_emails:
-            continue
-        ok = _create_contact(name, email)
-        if ok:
-            imported += 1
-            existing_emails.add(email.lower())
-            # If the CSV had a phone number, rewrite the just-created row
-            # through the richer update path so phone lands in CardDAV too.
-            if phone:
-                try:
-                    contacts = _fetch_contacts(force=True)
-                    created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
-                    if created and created.get("uid"):
-                        _update_contact(created["uid"], name, [email], [phone])
-                except Exception:
-                    pass
-        else:
-            failed += 1
-
-    if imported:
-        _contact_cache["fetched_at"] = None
-    return {"imported": imported, "failed": failed, "total": total}
-
-
-def _contacts_to_vcf(contacts: List[Dict]) -> str:
-    return "".join(
-        _build_vcard(
-            c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
-            "",
-            uid=c.get("uid") or str(uuid.uuid4()),
-            emails=c.get("emails") or [],
-            phones=c.get("phones") or [],
-        )
-        for c in contacts
-    )
-
-
-def _contacts_to_csv(contacts: List[Dict]) -> str:
-    out = io.StringIO()
-    writer = csv.writer(out)
-    writer.writerow(["name", "email", "phone"])
-    for c in contacts:
-        emails = c.get("emails") or [""]
-        phones = c.get("phones") or [""]
-        max_len = max(len(emails), len(phones), 1)
-        for i in range(max_len):
-            writer.writerow([
-                c.get("name") or "",
-                emails[i] if i < len(emails) else "",
-                phones[i] if i < len(phones) else "",
-            ])
-    return out.getvalue()
-
-
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
-    """Rewrite an existing contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        found = False
-        out = []
-        for c in contacts:
-            if c.get("uid") == uid:
-                # Preserve existing address when caller passes "" (only
-                # updating name/emails/phones, not touching address).
-                addr = address if address else c.get("address", "")
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
-                found = True
-            else:
-                out.append(c)
-        if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
-        _save_local_contacts(out)
-        return True
-
-    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
-    # Use the real resource href (handles externally-created contacts whose
-    # filename != UID); falls back to the <uid>.vcf guess.
-    try:
-        url = _resolve_resource_url(uid)
-        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-        r = httpx.put(
-            url,
-            data=vcard.encode("utf-8"),
-            headers={"Content-Type": "text/vcard; charset=utf-8"},
-            auth=auth,
-            timeout=10,
-        )
-        if r.status_code in (200, 201, 204):
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to update contact: {e}")
-        return False
-
-
-def _delete_contact(uid: str) -> bool:
-    """Delete a contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
-        _save_local_contacts(remaining)
-        return True
-
-    try:
-        url = _resolve_resource_url(uid)
-        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-        r = httpx.delete(url, auth=auth, timeout=10)
-        if r.status_code in (200, 204, 404):
-            # Invalidate cache so the next fetch sees the server truth.
-            _contact_cache["fetched_at"] = None
-            # Verify: force a fresh fetch and check the UID is actually gone.
-            # A 404 on the guessed URL ({uid}.vcf) can mean the contact
-            # lives at a different resource URL — the DELETE missed it but
-            # we'd silently report success. This check catches that.
-            fresh = _fetch_contacts(force=True)
-            still_there = any(c.get("uid") == uid for c in fresh)
-            if still_there:
-                logger.warning(
-                    f"CardDAV DELETE reported success for {uid} "
-                    f"but UID still present after re-fetch — "
-                    f"resource URL may differ from {redact_url(url)}"
-                )
-                return False
-            if r.status_code == 404:
-                logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
-            return True
-        logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to delete contact: {e}")
-        return False
-
-
-# ── Routes ──
-
-def setup_contacts_routes():
-    router = APIRouter(prefix="/api/contacts", tags=["contacts"])
-
-    @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
-        """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
-
-    @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
-        """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
-        if not q:
-            return {"results": []}
-        q_lower = q.lower()
-        results = []
-        for c in contacts:
-            if q_lower in c["name"].lower():
-                results.append(c)
-                continue
-            for em in c["emails"]:
-                if q_lower in em.lower():
-                    results.append(c)
-                    break
-        return {"results": results[:10]}
-
-    @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
-        """Add a new contact."""
-        name = (data.get("name") or "").strip()
-        email = (data.get("email") or "").strip()
-        phone = (data.get("phone") or "").strip()
-        address = (data.get("address") or "").strip()
-        if not email:
-            return {"success": False, "error": "Email required"}
-        # Check if already exists by email
-        if email:
-            contacts = _fetch_contacts()
-            for c in contacts:
-                if email.lower() in [e.lower() for e in c["emails"]]:
-                    return {"success": True, "message": "Already exists", "contact": c}
-        if not name:
-            name = email.split("@")[0]
-        create_params = inspect.signature(_create_contact).parameters
-        if len(create_params) >= 3:
-            ok = _create_contact(name, email, address)
-        else:
-            ok = _create_contact(name, email)
-        # If a phone was provided, do an immediate update to thread it
-        # through (the simple _create_contact signature only takes name +
-        # email + address; phones happen via update).
-        if ok and phone:
-            try:
-                fresh = _fetch_contacts(force=True)
-                created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
-                if created:
-                    _update_contact(
-                        created["uid"], name,
-                        created.get("emails", []),
-                        [phone],
-                        address,
-                    )
-            except Exception:
-                pass
-        return {"success": ok}
-
-    @router.post("/import")
-    async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
-        """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
-        # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
-        # in the JSON body) would otherwise reach .strip() and 500 with an
-        # AttributeError instead of degrading to a clean "no data" response.
-        text = str(data.get("vcf") or data.get("text") or "")
-        csv_text = str(data.get("csv") or "")
-        if text.strip():
-            if "BEGIN:VCARD" not in text.upper():
-                return {"success": False, "error": "No vCard data found"}
-            result = _import_vcards(text)
-        elif csv_text.strip():
-            result = _import_csv_contacts(csv_text)
-        else:
-            return {"success": False, "error": "No contact data found"}
-        result["success"] = result.get("imported", 0) > 0
-        return result
-
-    @router.get("/export")
-    async def export_contacts(
-        format: str = Query("vcf", pattern="^(vcf|csv)$"),
-        _admin: str = Depends(require_admin),
-    ):
-        """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
-        if format == "csv":
-            content = _contacts_to_csv(contacts)
-            media_type = "text/csv; charset=utf-8"
-            filename = "odysseus-contacts.csv"
-        else:
-            content = _contacts_to_vcf(contacts)
-            media_type = "text/vcard; charset=utf-8"
-            filename = "odysseus-contacts.vcf"
-        return Response(
-            content=content,
-            media_type=media_type,
-            headers={"Content-Disposition": f'attachment; filename="{filename}"'},
-        )
-
-    @router.get("/config")
-    async def get_config(_admin: str = Depends(require_admin)):
-        cfg = _get_carddav_config()
-        # Mask password
-        if cfg["password"]:
-            cfg["password"] = "***"
-        return cfg
-
-    @router.put("/config")
-    async def update_config(data: dict, _admin: str = Depends(require_admin)):
-        settings = _load_settings()
-        for key in ("carddav_url", "carddav_username", "carddav_password"):
-            if key in data:
-                if key == "carddav_url" and str(data[key] or "").strip():
-                    try:
-                        settings[key] = _validate_carddav_url(data[key])
-                    except ValueError as e:
-                        raise HTTPException(400, str(e))
-                else:
-                    value = data[key]
-                    if key == "carddav_password" and value:
-                        from src.secret_storage import encrypt
-                        value = encrypt(value)
-                    settings[key] = value
-        _save_settings(settings)
-        # Force re-fetch
-        _contact_cache["fetched_at"] = None
-        return {"success": True}
-
-    @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
-        """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
-        return {"success": True}
-
-    # NOTE: the /{uid} routes are declared LAST so the literal paths above
-    # (/list, /search, /add, /config) win — otherwise PUT /config would
-    # match PUT /{uid} with uid="config".
-    @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
-        """Edit an existing contact — name / emails / phones / address."""
-        name = (data.get("name") or "").strip()
-        emails = data.get("emails")
-        phones = data.get("phones")
-        if emails is None and data.get("email"):
-            emails = [data["email"]]
-        emails = [e.strip() for e in (emails or []) if e and e.strip()]
-        phones = [p.strip() for p in (phones or []) if p and p.strip()]
-        address = (data.get("address") or "").strip()
-        if not name and not emails and not address:
-            return {"success": False, "error": "Name, email, or address required"}
-        if not name and emails:
-            name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones, address)
-        return {"success": ok}
-
-    @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
-        """Delete a contact by UID."""
-        if not uid:
-            return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
-        return {"success": ok}
-
-    return router

routes/cookbook_helpers.py

@@ -439,15 +439,30 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "                if f.is_file(): nf += 1; sz += f.stat().st_size",
         "                if f.name.endswith('.incomplete'): ic = True",
         "        snap = os.path.join(cache, d, 'snapshots')",
-        "        # Windows HF cache stores files directly in snapshots/; blobs/ may be empty.",
+        "        def snapshot_size():",
-        "        # Fallback: scan snapshots for real files when blobs yielded nothing.",
+        "            total, count, incomplete = 0, 0, False",
-        "        if sz == 0 and os.path.isdir(snap):",
+        "            seen_real = set()",
         "            for sd in os.listdir(snap):",
         "                sf = os.path.join(snap, sd)",
         "                if not os.path.isdir(sf): continue",
-        "                for f in os.scandir(sf):",
+        "                for root, dirs, fns in safe_walk(sf):",
-        "                    if f.is_file(): nf += 1; sz += f.stat().st_size",
+        "                    for fn in fns:",
-        "                    if f.name.endswith('.incomplete'): ic = True",
+        "                        fp = os.path.join(root, fn)",
+        "                        if fn.endswith('.incomplete'): incomplete = True",
+        "                        try:",
+        "                            real = os.path.realpath(fp)",
+        "                            if real in seen_real: continue",
+        "                            seen_real.add(real)",
+        "                            total += os.path.getsize(real)",
+        "                            count += 1",
+        "                        except Exception:",
+        "                            pass",
+        "            return total, count, incomplete",
+        "        # Some HF caches (macOS/MLX/Xet-style) keep blobs elsewhere or expose",
+        "        # snapshot symlinks only. Size snapshots too when blob accounting is empty.",
+        "        if sz == 0 and os.path.isdir(snap):",
+        "            sz2, nf2, ic2 = snapshot_size()",
+        "            sz, nf, ic = sz2, nf2, ic or ic2",
         "        is_diffusion = False; gguf_files = []",
         "        if os.path.isdir(snap):",
         "            for sd in os.listdir(snap):",
@@ -471,7 +486,18 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "    add('/app/.cache/huggingface/hub')",
         f"    add({add_hf_cache!r})" if add_hf_cache else "",
         "    return candidates",
+        "def normalize_model_dir(p):",
+        "    p = os.path.expanduser((p or '').strip())",
+        "    if not p: return p",
+        "    if os.path.isdir(p) or os.path.isabs(p): return p",
+        "    # Users often paste Linux absolute paths without the leading slash.",
+        "    # Treat home/<user>/... as /home/<user>/... so remote scans work.",
+        "    if p.startswith(('home/', 'mnt/', 'media/', 'data/', 'opt/', 'srv/', 'var/')):",
+        "        prefixed = '/' + p",
+        "        if os.path.isdir(prefixed): return prefixed",
+        "    return p",
         "def scan_dir(p):",
+        "    p = normalize_model_dir(p)",
         "    if not os.path.isdir(p) or not safe_path(p): return",
         "    for d in sorted(os.listdir(p)):",
         "        if d.startswith('.'): continue",
@@ -541,7 +567,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "scan_ollama()",
     ]
     for model_dir in model_dirs or []:
-        lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))")
+        lines.append(f"scan_dir({model_dir!r})")
     lines.append("print(json.dumps(models))")
     return "\n".join(lines) + "\n"
 
@@ -1273,13 +1299,15 @@ def _diagnose_serve_output(text: str) -> dict | None:
             [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
         ),
         (
-            r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
+            r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
-            r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
+            r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
+            r"Could not load any common_ops library|"
             r"Please ensure sgl_kernel is properly installed",
-            "SGLang native dependencies are missing on this server.",
+            "SGLang native kernel/runtime is missing or mismatched on this server.",
             [
+                {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
                 {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
-                {"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
+                {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
             ],
         ),
         (
@@ -1287,6 +1315,19 @@ def _diagnose_serve_output(text: str) -> dict | None:
             "SGLang is not installed or not in PATH on this server.",
             [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
         ),
+        (
+            r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
+            "MLX LM is not installed on this server.",
+            [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
+        ),
+        (
+            r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
+            "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
+            [
+                {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
+                {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
+            ],
+        ),
         # System build deps come BEFORE the generic llama.cpp catch-all so
         # cmake / build-essential / git missing → a specific OS-package
         # remediation instead of "install llama-cpp-python[server]" (which

routes/document_routes.py

@@ -12,6 +12,7 @@ from core.database import SessionLocal, Document, DocumentVersion
 from core.database import Session as DbSession
 from src.auth_helpers import get_current_user, _auth_disabled
 from src.constants import MAIL_ATTACHMENTS_DIR
+from src.upload_handler import reserve_upload_references
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +55,18 @@ def _library_language_for_document(doc: Document) -> str:
     return doc.language or "text"
 
 
+def _email_source_key(content: str) -> tuple[str, str]:
+    """Return the source email identity embedded in an email draft document."""
+    import re
+
+    text = content or ""
+    uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text)
+    folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text)
+    uid = (uid_m.group(1).strip() if uid_m else "")
+    folder = (folder_m.group(1).strip() if folder_m else "INBOX")
+    return uid, folder
+
+
 from routes.document_helpers import (
     DocumentCreate, DocumentUpdate, DocumentPatch,
     _doc_to_dict, _version_to_dict,
@@ -66,6 +79,14 @@ from routes.document_helpers import (
 def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
     router = APIRouter(tags=["documents"])
 
+    def _reserve_document_uploads(user: Optional[str], content: str) -> None:
+        missing_id = reserve_upload_references(upload_handler, user, content)
+        if missing_id:
+            raise HTTPException(
+                409,
+                f"Referenced upload is no longer available: {missing_id}",
+            )
+
     def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]):
         if upload_handler is None:
             return None
@@ -99,24 +120,63 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 # the existing lenient path.
                 session = _get_session_or_404(db, req.session_id, user)
 
-            doc_id = str(uuid.uuid4())
-            ver_id = str(uuid.uuid4())
-
             # If no language was supplied (e.g. cloning a doc whose language
             # was never set), detect it from the content rather than storing
             # NULL — which made the editor fall back to plain text. Defaults
             # to markdown for prose.
             language = req.language
             if not language:
-                from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language
+                from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content
                 language = _sniff_doc_language(req.content)
             else:
-                from src.agent_tools.document_tools import _looks_like_email_document
+                from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content
             if _looks_like_email_document(req.content, req.title):
                 language = "email"
 
+            _reserve_document_uploads(user, req.content)
             _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
 
+            # Reply drafts are keyed to the source email. If a UI/tool path tries
+            # to create a second draft for the same email in the same chat,
+            # update the existing draft instead so quoted thread history stays
+            # attached to the visible document.
+            if language == "email" and req.session_id:
+                source_uid, source_folder = _email_source_key(req.content)
+                if source_uid:
+                    candidates = (
+                        db.query(Document)
+                        .filter(Document.session_id == req.session_id)
+                        .filter(Document.is_active == True)
+                        .filter(Document.language == "email")
+                        .order_by(Document.updated_at.desc())
+                        .limit(25)
+                        .all()
+                    )
+                    for existing in candidates:
+                        old_uid, old_folder = _email_source_key(existing.current_content or "")
+                        if old_uid != source_uid or old_folder != source_folder:
+                            continue
+                        merged = _coerce_email_document_content(existing.current_content or "", req.content)
+                        if existing.current_content != merged:
+                            new_ver = (existing.version_count or 1) + 1
+                            existing.current_content = merged
+                            existing.title = req.title or existing.title
+                            existing.version_count = new_ver
+                            db.add(DocumentVersion(
+                                id=str(uuid.uuid4()),
+                                document_id=existing.id,
+                                version_number=new_ver,
+                                content=merged,
+                                summary="Updated existing email draft",
+                                source="user",
+                            ))
+                            db.commit()
+                            db.refresh(existing)
+                        return _doc_to_dict(existing)
+
+            doc_id = str(uuid.uuid4())
+            ver_id = str(uuid.uuid4())
+
             doc = Document(
                 id=doc_id,
                 session_id=req.session_id,
@@ -570,12 +630,24 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 raise HTTPException(404, "Document not found")
             _verify_doc_owner(db, doc, user)
 
+            incoming_content = req.content
+            from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document
+            is_email_doc = (
+                (doc.language or "").lower() == "email"
+                or _looks_like_email_document(doc.current_content or "", doc.title or "")
+                or _looks_like_email_document(req.content or "", doc.title or "")
+            )
+            if is_email_doc:
+                incoming_content = _coerce_email_document_content(doc.current_content or "", req.content)
+                doc.language = "email"
+
             # Skip if content is identical unless the caller explicitly wants
             # a checkpoint version from the current editor state.
-            if doc.current_content == req.content and not req.force_version:
+            if doc.current_content == incoming_content and not req.force_version:
                 return _doc_to_dict(doc)
 
-            _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
+            _reserve_document_uploads(user, incoming_content)
+            _assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler)
 
             # Check if we can coalesce with the latest version
             latest_ver = db.query(DocumentVersion).filter(
@@ -591,7 +663,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 age = (now - ver_time).total_seconds()
                 if age < VERSION_COALESCE_SECONDS:
                     # Update the existing version in-place
-                    latest_ver.content = req.content
+                    latest_ver.content = incoming_content
                     latest_ver.created_at = now
                     if req.summary:
                         latest_ver.summary = req.summary
@@ -603,14 +675,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                     id=str(uuid.uuid4()),
                     document_id=doc_id,
                     version_number=new_ver,
-                    content=req.content,
+                    content=incoming_content,
                     summary=req.summary or "Manual edit",
                     source="user",
                 )
                 doc.version_count = new_ver
                 db.add(ver)
 
-            doc.current_content = req.content
+            doc.current_content = incoming_content
             db.commit()
             db.refresh(doc)
             return _doc_to_dict(doc)

routes/email_helpers.py

@@ -349,7 +349,7 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
             row = db.query(_EA).filter(_EA.id == account_id).first()
             if row is None:
                 raise HTTPException(404, "Account not found")
-            if row.owner and row.owner != owner:
+            if not _account_visible_to_owner(row, owner):
                 # Treat as 404 (not 403) so we don't leak existence.
                 raise HTTPException(404, "Account not found")
         finally:
@@ -362,6 +362,26 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
         logger.error(f"Account-owner check failed: {e}")
         raise HTTPException(503, "Account check failed")
 
+
+def _account_visible_to_owner(row, owner: str) -> bool:
+    """Whether an authenticated `owner` may act on this EmailAccount row.
+
+    Mirrors the SQL predicate in `_get_email_config`'s
+    `_owner_or_matching_legacy_account`: a caller sees an account they own, or a
+    legacy owner-less account (owner NULL/"") only when its own mailbox
+    (`imap_user` / `from_address`) is the caller's. `email_accounts` is the one
+    owner-scoped table deliberately left out of the legacy-owner migration
+    backfill, so ownerless rows persist on multi-user deploys — making this the
+    gate that keeps one tenant off another's imported mailbox and its decrypted
+    IMAP/SMTP credentials."""
+    row_owner = getattr(row, "owner", None) or ""
+    if row_owner:
+        return row_owner == owner
+    return owner in {
+        getattr(row, "imap_user", None) or "",
+        getattr(row, "from_address", None) or "",
+    }
+
 def _q(name: str) -> str:
     """Quote an IMAP mailbox name. Defensive: escapes `\\` and `"` and wraps
     in double quotes so user-supplied folder names with spaces or quotes can't
@@ -903,12 +923,13 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
         try:
             if account_id:
                 row = db.query(_EA).filter(_EA.id == account_id, _EA.enabled == True).first()  # noqa: E712
-                # If the resolved row belongs to a different owner, treat as
+                # If the resolved row isn't visible to this owner, treat as
                 # not-found rather than silently serving it. This is a defense
                 # in depth — `require_owner` already calls `_assert_owns_account`
                 # for query-param account_ids, but other callers (cookbook
-                # rules, scheduled poller) may not.
+                # rules, scheduled poller) may not. Ownerless legacy rows are
-                if row is not None and owner and row.owner and row.owner != owner:
+                # only visible on a mailbox match, same as the fallback below.
+                if row is not None and owner and not _account_visible_to_owner(row, owner):
                     row = None
             # Fallback path — restrict to this owner's accounts so we don't
             # leak another user's default mailbox to an unconfigured user.

routes/email_pollers.py

@@ -1068,6 +1068,28 @@ def _scheduled_poll_once() -> dict:
         for r in rows:
             sid = r[0]
             try:
+                # Atomically claim this row before doing any work. Two
+                # pollers can race here (the in-process asyncio task and an
+                # externally cron-driven `odysseus-mail poll-scheduled`, or
+                # an admin running the CLI manually alongside the in-process
+                # one despite the ODYSSEUS_INPROCESS_POLLERS=0 guidance) -
+                # both can SELECT the same 'pending' row before either has
+                # updated its status. The UPDATE...WHERE status='pending' is
+                # the atomicity boundary: only the poller whose UPDATE
+                # actually changes a row (rowcount == 1) proceeds to send;
+                # a loser sees rowcount == 0 and skips it instead of sending
+                # a duplicate.
+                claim_conn = sqlite3.connect(SCHEDULED_DB)
+                claim_cur = claim_conn.execute(
+                    "UPDATE scheduled_emails SET status='sending' WHERE id=? AND status='pending'",
+                    (sid,),
+                )
+                claim_conn.commit()
+                claimed = claim_cur.rowcount == 1
+                claim_conn.close()
+                if not claimed:
+                    continue
+
                 attachments = json.loads(r[8] or "[]")
                 row_account_id = r[9] if len(r) > 9 else None
                 odysseus_kind = r[10] if len(r) > 10 else "scheduled"

routes/email_routes.py

@@ -23,6 +23,8 @@ import smtplib
 import json
 import re
 import html
+import io
+import zipfile
 from html.parser import HTMLParser as _HTMLParser
 import logging
 import uuid
@@ -33,7 +35,7 @@ from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
 from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, StreamingResponse
 from src.constants import DATA_DIR
 
 from src.llm_core import llm_call_async
@@ -65,6 +67,14 @@ ODYSSEUS_MAIL_ORIGIN = "odysseus-ui"
 EMAIL_READ_ATTACHMENT_VERSION = 2
 
 
+def _safe_attachment_zip_name(name: str, fallback: str) -> str:
+    """Return a zip entry filename without path traversal or empty names."""
+    base = Path(str(name or "")).name.strip() or fallback
+    base = re.sub(r"[\x00-\x1f\x7f]+", "_", base)
+    base = base.replace("/", "_").replace("\\", "_").strip(". ") or fallback
+    return base[:180] or fallback
+
+
 def _coerce_port(value, default):
     """Coerce a user-supplied port to int.
 
@@ -484,23 +494,37 @@ def _email_index_rows(owner: str, account_id: str | None, folder: str, uids: lis
     return out
 
 
-def _email_index_search(owner: str, account_id: str | None, folder: str, query: str, limit: int) -> tuple[list[dict], int, str | None]:
+def _email_index_search(owner: str, account_id: str | None, folder: str, query: str, limit: int, global_search: bool = True) -> tuple[list[dict], int, str | None]:
     q = (query or "").strip()
     if not q:
         return [], 0, None
     limit = max(1, min(int(limit or 50), 200))
     account_key = _account_cache_key(account_id, owner)
-    like = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
     folder_clause = ""
     params: list = [owner or "", account_key]
     # Searching from INBOX should feel global for Gmail-style accounts,
     # because users expect archived/labelled mail to show up too. The
     # local index only contains folders that have been warmed/listed, so
     # this remains a best-effort fast path; IMAP is still the fallback.
-    if (folder or "").upper() != "INBOX":
+    if not global_search or (folder or "").upper() != "INBOX":
         folder_clause = "AND folder=?"
         params.append(folder)
-    params.extend([like, like, like, like, like])
+    terms = _email_search_terms(q)
+    if not terms:
+        return [], 0, None
+    term_clause = " AND ".join([
+        """(
+                    subject LIKE ? ESCAPE '\\' OR
+                    from_name LIKE ? ESCAPE '\\' OR
+                    from_address LIKE ? ESCAPE '\\' OR
+                    to_text LIKE ? ESCAPE '\\' OR
+                    cc_text LIKE ? ESCAPE '\\'
+                  )"""
+        for _ in terms
+    ])
+    for term in terms:
+        like = "%" + term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+        params.extend([like, like, like, like, like])
     try:
         conn = _sql3.connect(SCHEDULED_DB)
         try:
@@ -509,13 +533,7 @@ def _email_index_search(owner: str, account_id: str | None, folder: str, query:
                 SELECT COUNT(*), MAX(updated_at)
                 FROM email_message_index
                 WHERE owner=? AND account_key=? {folder_clause}
-                  AND (
+                  AND {term_clause}
-                    subject LIKE ? ESCAPE '\\' OR
-                    from_name LIKE ? ESCAPE '\\' OR
-                    from_address LIKE ? ESCAPE '\\' OR
-                    to_text LIKE ? ESCAPE '\\' OR
-                    cc_text LIKE ? ESCAPE '\\'
-                  )
                 """,
                 params,
             ).fetchone()
@@ -529,13 +547,7 @@ def _email_index_search(owner: str, account_id: str | None, folder: str, query:
                        folder
                 FROM email_message_index
                 WHERE owner=? AND account_key=? {folder_clause}
-                  AND (
+                  AND {term_clause}
-                    subject LIKE ? ESCAPE '\\' OR
-                    from_name LIKE ? ESCAPE '\\' OR
-                    from_address LIKE ? ESCAPE '\\' OR
-                    to_text LIKE ? ESCAPE '\\' OR
-                    cc_text LIKE ? ESCAPE '\\'
-                  )
                 ORDER BY date_epoch DESC
                 LIMIT ?
                 """,
@@ -573,6 +585,64 @@ def _email_index_search(owner: str, account_id: str | None, folder: str, query:
     return emails, total, (total_row or [None, None])[1]
 
 
+def _email_search_terms(query: str) -> list[str]:
+    q = (query or "").strip()
+    if not q:
+        return []
+    # Preserve quoted phrases, then split the rest. This makes:
+    #   honda insurance -> honda AND insurance
+    #   "Yoko Honda" insurance -> "Yoko Honda" AND insurance
+    # The cap avoids creating huge IMAP expressions from pasted paragraphs.
+    parts = []
+    consumed = []
+    for m in re.finditer(r'"([^"]{1,120})"', q):
+        phrase = m.group(1).strip()
+        if phrase:
+            parts.append(phrase)
+        consumed.append((m.start(), m.end()))
+    remainder = q
+    for start, end in reversed(consumed):
+        remainder = remainder[:start] + " " + remainder[end:]
+    parts.extend(re.findall(r"[^\s,;]+", remainder))
+    out = []
+    seen = set()
+    for p in parts:
+        p = p.strip().strip('"').strip()
+        if len(p) < 2:
+            continue
+        key = p.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(p)
+        if len(out) >= 6:
+            break
+    return out
+
+
+def _imap_or_many(keys: list[str]) -> str:
+    if not keys:
+        return "ALL"
+    expr = keys[0]
+    for key in keys[1:]:
+        expr = f"OR ({expr}) ({key})"
+    return expr
+
+
+def _email_imap_search_criteria(query: str) -> str:
+    terms = _email_search_terms(query)
+    if not terms:
+        return "ALL"
+    term_exprs = []
+    for term in terms:
+        q = _imap_search_quote(term)
+        # Search both sides of the conversation, plus subject and body. The
+        # older route only searched FROM/SUBJECT/TEXT, so recipient searches
+        # and many sent-message searches felt broken.
+        term_exprs.append(f"({_imap_or_many([f'FROM {q}', f'TO {q}', f'CC {q}', f'SUBJECT {q}', f'TEXT {q}'])})")
+    return "(" + " ".join(term_exprs) + ")"
+
+
 def _email_index_upsert(owner: str, account_id: str | None, folder: str, emails: list[dict]):
     if not emails:
         return
@@ -853,29 +923,32 @@ def _resolve_send_config(account_id: str | None = None, owner: str = "") -> dict
 
 
 def _store_email_flag(conn, uid: str, flag: str, add: bool = True) -> bool:
+    # imaplib's plain store() takes a message SEQUENCE NUMBER, not a UID, so the
+    # old `else` fallback flagged whichever message happened to occupy sequence
+    # position == the UID value. When the UID isn't present, fail safe (callers
+    # surface "Email not found") rather than touch an unrelated message.
+    if not _uid_exists(conn, uid):
+        return False
     op = "+FLAGS" if add else "-FLAGS"
-    if _uid_exists(conn, uid):
+    status, _ = conn.uid("STORE", _uid_bytes(uid), op, flag)
-        status, _ = conn.uid("STORE", _uid_bytes(uid), op, flag)
-    else:
-        status, _ = conn.store(_uid_bytes(uid), op, flag)
     return status == "OK"
 
 
 def _move_email_message(conn, uid: str, dest: str, role: str = "") -> bool:
     dest = _resolve_mail_folder(conn, dest, role or _folder_role_from_name(dest))
-    if _uid_exists(conn, uid):
+    # copy()/store() are SEQUENCE-NUMBER commands; using them with a UID (the old
-        status, _ = conn.uid("MOVE", _uid_bytes(uid), _q(dest))
+    # `else` branch) copied + \Deleted-flagged the wrong message and then
-        if status == "OK":
+    # expunge() permanently removed it. There is no valid case where treating a
-            return True
+    # UID as a sequence number is correct, so fail safe when the UID is absent.
-        status, _ = conn.uid("COPY", _uid_bytes(uid), _q(dest))
+    if not _uid_exists(conn, uid):
-        if status != "OK":
+        return False
-            return False
+    status, _ = conn.uid("MOVE", _uid_bytes(uid), _q(dest))
-        status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "\\Deleted")
+    if status == "OK":
-    else:
+        return True
-        status, _ = conn.copy(_uid_bytes(uid), _q(dest))
+    status, _ = conn.uid("COPY", _uid_bytes(uid), _q(dest))
-        if status != "OK":
+    if status != "OK":
-            return False
+        return False
-        status, _ = conn.store(_uid_bytes(uid), "+FLAGS", "\\Deleted")
+    status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "\\Deleted")
     if status == "OK":
         conn.expunge()
         return True
@@ -2049,6 +2122,7 @@ def setup_email_routes():
         limit: int = Query(50),
         account_id: str | None = Query(None),
         local_only: bool = Query(False),
+        scope: str = Query("all"),
         owner: str = Depends(require_owner),
     ):
         """Search emails server-side via IMAP SEARCH. Matches subject, from, or body text.
@@ -2062,9 +2136,10 @@ def setup_email_routes():
         # CRLF in q would terminate the IMAP command early — reject defensively.
         if "\r" in q or "\n" in q:
             raise HTTPException(400, "Invalid query")
+        global_search = (scope or "all").lower() != "folder"
         indexed_response = None
         try:
-            indexed_emails, indexed_total, indexed_at = _email_index_search(owner, account_id, folder, q, limit)
+            indexed_emails, indexed_total, indexed_at = _email_index_search(owner, account_id, folder, q, limit, global_search=global_search)
             indexed_response = {
                 "emails": indexed_emails,
                 "total": indexed_total,
@@ -2083,7 +2158,7 @@ def setup_email_routes():
                 # If the user asked for INBOX, try to upgrade to All Mail —
                 # one folder == every email on Gmail-class servers.
                 effective_folder = folder
-                if (folder or "").upper() == "INBOX":
+                if global_search and (folder or "").upper() == "INBOX":
                     try:
                         status, folder_lines = conn.list()
                         if status == "OK" and folder_lines:
@@ -2102,12 +2177,18 @@ def setup_email_routes():
                         pass
                 conn.select(_q(effective_folder), readonly=True)
 
-                # Escape backslash and quote for the IMAP-SEARCH quoted-string.
+                search_cmd = _email_imap_search_criteria(q)
-                q_escaped = q.replace('\\', '\\\\').replace('"', '\\"')
-                search_cmd = f'(OR OR FROM "{q_escaped}" SUBJECT "{q_escaped}" TEXT "{q_escaped}")'
 
                 status, data = _imap_uid_search(conn, search_cmd)
                 if status != "OK" or not data[0]:
+                    if indexed_response and indexed_response.get("emails"):
+                        indexed_response["fallback"] = True
+                        indexed_response["source"] = "index"
+                        indexed_response["sync"] = {
+                            **(indexed_response.get("sync") or {}),
+                            "fallback_reason": "imap_empty" if status == "OK" else "imap_search_failed",
+                        }
+                        return indexed_response
                     return {
                         "emails": [],
                         "total": 0,
@@ -2530,6 +2611,59 @@ def setup_email_routes():
             logger.error(f"Failed to download attachment {uid}/{index}: {e}")
             return {"error": "Mail operation failed"}
 
+    @router.get("/attachments-download/{uid}")
+    async def download_all_attachments(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)):
+        """Download all visible attachments for an email as a zip archive."""
+        try:
+            with _imap(account_id, owner=owner) as conn:
+                conn.select(_q(folder), readonly=True)
+                status, msg_data = _imap_uid_fetch(conn, uid, "(RFC822)")
+            if status != "OK":
+                raise HTTPException(status_code=404, detail="Email not found")
+            raw = msg_data[0][1]
+            msg = email_mod.message_from_bytes(raw)
+            attachments = [
+                att for att in _list_attachments_from_msg(msg)
+                if not _is_likely_signature_image_attachment(att)
+            ]
+            if not attachments:
+                raise HTTPException(status_code=404, detail="No downloadable attachments")
+
+            target_dir = attachment_extract_dir(folder, uid)
+            zip_buf = io.BytesIO()
+            used_names: dict[str, int] = {}
+            with zipfile.ZipFile(zip_buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+                for att in attachments:
+                    idx = att.get("index")
+                    if idx is None:
+                        continue
+                    filepath = _extract_attachment_to_disk(msg, int(idx), target_dir)
+                    if not filepath or not Path(filepath).exists():
+                        continue
+                    fallback = f"attachment-{idx}"
+                    arcname = _safe_attachment_zip_name(att.get("filename") or Path(filepath).name, fallback)
+                    stem = Path(arcname).stem
+                    suffix = Path(arcname).suffix
+                    seen = used_names.get(arcname, 0)
+                    used_names[arcname] = seen + 1
+                    if seen:
+                        arcname = f"{stem}-{seen + 1}{suffix}"
+                    zf.write(str(filepath), arcname)
+            zip_buf.seek(0)
+            if not zip_buf.getbuffer().nbytes:
+                raise HTTPException(status_code=404, detail="No downloadable attachments")
+            zip_name = _safe_attachment_zip_name(f"email-{uid}-attachments.zip", "attachments.zip")
+            return StreamingResponse(
+                zip_buf,
+                media_type="application/zip",
+                headers={"Content-Disposition": f'attachment; filename="{zip_name}"'},
+            )
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Failed to download attachments zip {uid}: {e}")
+            raise HTTPException(status_code=500, detail="Mail operation failed")
+
     @router.get("/inline-image/{uid}")
     async def inline_image(
         uid: str,
@@ -3150,6 +3284,155 @@ def setup_email_routes():
             logger.error(f"Failed to upload attachment: {e}")
             return {"success": False, "error": "Mail operation failed"}
 
+    def _safe_compose_filename(name: str, fallback: str = "attachment") -> str:
+        safe_name = re.sub(r"[^\w\s\-.]", "_", Path(str(name or fallback)).name).strip(". ")[:180]
+        return safe_name or fallback
+
+    def _stage_compose_bytes(filename: str, content: bytes) -> dict:
+        if len(content) > EMAIL_COMPOSE_UPLOAD_MAX_BYTES:
+            raise HTTPException(status_code=413, detail="Attachment too large")
+        safe_name = _safe_compose_filename(filename)
+        token = f"{uuid.uuid4().hex}_{safe_name}"
+        filepath = COMPOSE_UPLOADS_DIR / token
+        with open(filepath, "wb") as f:
+            f.write(content)
+        return {"success": True, "token": token, "filename": safe_name, "size": len(content)}
+
+    def _stage_compose_file(filename: str, src: Path) -> dict:
+        if not src.exists() or not src.is_file():
+            raise HTTPException(status_code=404, detail="File not found")
+        size = src.stat().st_size
+        if size > EMAIL_COMPOSE_UPLOAD_MAX_BYTES:
+            raise HTTPException(status_code=413, detail="Attachment too large")
+        safe_name = _safe_compose_filename(filename)
+        token = f"{uuid.uuid4().hex}_{safe_name}"
+        dest = COMPOSE_UPLOADS_DIR / token
+        import shutil as _shutil
+        _shutil.copyfile(str(src), str(dest))
+        return {"success": True, "token": token, "filename": safe_name, "size": size}
+
+    def _load_odysseus_attachment_source(db, kind: str, item_id: str, owner: str):
+        from core.database import Document as _Doc, GalleryImage as _GI
+        from core.database import Session as _Sess
+
+        if kind == "document":
+            doc = db.query(_Doc).filter(_Doc.id == item_id, _Doc.is_active == True).first()
+            if not doc:
+                raise HTTPException(status_code=404, detail="Document not found")
+            if owner:
+                if doc.owner and doc.owner != owner:
+                    raise HTTPException(status_code=404, detail="Document not found")
+                if not doc.owner and doc.session_id:
+                    sess = db.query(_Sess).filter(_Sess.id == doc.session_id).first()
+                    if sess and sess.owner and sess.owner != owner:
+                        raise HTTPException(status_code=404, detail="Document not found")
+            lang = (doc.language or "text").strip().lower()
+            ext = {
+                "markdown": "md",
+                "email": "eml",
+                "json": "json",
+                "yaml": "yaml",
+                "yml": "yml",
+                "html": "html",
+                "csv": "csv",
+                "xml": "xml",
+                "text": "txt",
+            }.get(lang, "txt")
+            base = _safe_compose_filename(doc.title or "document", "document")
+            if not base.lower().endswith(f".{ext}"):
+                base = f"{base}.{ext}"
+            return {"filename": base, "content": (doc.current_content or "").encode("utf-8")}
+
+        if kind == "gallery":
+            img = db.query(_GI).filter(_GI.id == item_id, _GI.is_active == True).first()
+            if not img:
+                raise HTTPException(status_code=404, detail="Image not found")
+            if owner and img.owner and img.owner != owner:
+                raise HTTPException(status_code=404, detail="Image not found")
+            from routes.gallery.gallery_routes import _gallery_image_path
+            src = _gallery_image_path(img.filename)
+            if not src.exists() or not src.is_file():
+                raise HTTPException(status_code=404, detail="Image file not found")
+            return {"filename": _safe_compose_filename(img.filename or "gallery-image.png"), "path": src}
+
+        raise HTTPException(status_code=400, detail="Unknown attachment kind")
+
+    @router.post("/compose-from-odysseus")
+    async def compose_from_odysseus(data: dict, owner: str = Depends(require_owner)):
+        """Stage an Odysseus document or gallery image as a compose upload."""
+        kind = str(data.get("kind") or "").strip().lower()
+        item_id = str(data.get("id") or "").strip()
+        if kind not in {"document", "gallery"} or not item_id:
+            raise HTTPException(status_code=400, detail="Expected kind and id")
+        try:
+            from core.database import SessionLocal as _SL
+
+            db = _SL()
+            try:
+                src = _load_odysseus_attachment_source(db, kind, item_id, owner)
+                if "path" in src:
+                    return _stage_compose_file(src["filename"], src["path"])
+                return _stage_compose_bytes(src["filename"], src["content"])
+            finally:
+                db.close()
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Failed to stage Odysseus attachment {kind}/{item_id}: {e}")
+            return {"success": False, "error": "Mail operation failed"}
+
+    @router.post("/compose-from-odysseus-zip")
+    async def compose_from_odysseus_zip(data: dict, owner: str = Depends(require_owner)):
+        """Stage several Odysseus documents/gallery images as one zip attachment."""
+        raw_items = data.get("items") or []
+        if not isinstance(raw_items, list) or not raw_items:
+            raise HTTPException(status_code=400, detail="Expected items")
+        if len(raw_items) > 100:
+            raise HTTPException(status_code=400, detail="Too many attachments")
+        try:
+            from core.database import SessionLocal as _SL
+
+            db = _SL()
+            try:
+                buf = io.BytesIO()
+                used_names: dict[str, int] = {}
+
+                def unique_name(name: str) -> str:
+                    safe = _safe_attachment_zip_name(name, "attachment")
+                    stem = Path(safe).stem or "attachment"
+                    suffix = Path(safe).suffix
+                    idx = used_names.get(safe, 0)
+                    used_names[safe] = idx + 1
+                    if idx == 0:
+                        return safe
+                    return f"{stem}-{idx + 1}{suffix}"
+
+                with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+                    for item in raw_items:
+                        if not isinstance(item, dict):
+                            continue
+                        kind = str(item.get("kind") or "").strip().lower()
+                        item_id = str(item.get("id") or "").strip()
+                        if kind not in {"document", "gallery"} or not item_id:
+                            continue
+                        src = _load_odysseus_attachment_source(db, kind, item_id, owner)
+                        zname = unique_name(src["filename"])
+                        if "path" in src:
+                            zf.write(src["path"], arcname=zname)
+                        else:
+                            zf.writestr(zname, src["content"])
+                content = buf.getvalue()
+                if not content:
+                    raise HTTPException(status_code=400, detail="No valid attachments")
+                return _stage_compose_bytes("odysseus-attachments.zip", content)
+            finally:
+                db.close()
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Failed to stage Odysseus zip attachment: {e}")
+            return {"success": False, "error": "Mail operation failed"}
+
     @router.post("/compose-from-attachment/{uid}/{index}")
     async def compose_from_attachment(
         uid: str,
@@ -4423,7 +4706,9 @@ def setup_email_routes():
         cfg["email_auto_tag"] = bool(settings.get("email_auto_tag", False))
         cfg["email_auto_spam"] = bool(settings.get("email_auto_spam", False))
         cfg["email_auto_calendar"] = bool(settings.get("email_auto_calendar", False))
-        cfg["email_auto_translate"] = bool(settings.get("email_auto_translate", False))
+        # Email translation is owned by the background task now; opening an email
+        # should not trigger reader-side auto-translation from Settings.
+        cfg["email_auto_translate"] = False
         cfg["email_translate_language"] = settings.get("email_translate_language", "English")
         return cfg
 
@@ -4438,11 +4723,9 @@ def setup_email_routes():
         """
         # Automation flags stay in settings.json (they're global, not per-account)
         settings = _load_settings()
-        for key in ["email_auto_summarize", "email_auto_reply", "email_auto_tag", "email_auto_spam", "email_auto_calendar", "email_auto_translate"]:
+        for key in ["email_auto_summarize", "email_auto_reply", "email_auto_tag", "email_auto_spam", "email_auto_calendar"]:
             if key in data:
                 settings[key] = data[key]
-        if "email_translate_language" in data:
-            settings["email_translate_language"] = (data.get("email_translate_language") or "English").strip() or "English"
         _save_settings(settings)
 
         # Credentials go into the default account row

routes/history/__init__.py

@@ -0,0 +1,5 @@
+"""History route domain package (slice 2d, #4082/#4071).
+
+Contains history_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/history_routes.py re-exports from here.
+"""

routes/history/history_routes.py

@@ -0,0 +1,794 @@
+"""History routes — session history, truncation, fork, conversation topics."""
+
+import json
+import uuid
+import logging
+import re
+from typing import Dict, Any, Optional
+
+from fastapi import APIRouter, Request, HTTPException
+
+from core.models import ChatMessage
+from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
+from src.auth_helpers import effective_user
+from src.topic_analyzer import analyze_topics
+from src.upload_handler import reserve_message_upload_references
+from routes.session_routes import (
+    _message_role,
+    _message_text,
+    _reject_compact_during_active_run,
+    _verify_session_owner,
+)
+
+logger = logging.getLogger(__name__)
+
+_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
+_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
+
+
+def _history_display_content(content: Any) -> Any:
+    """Return a lightweight browser-display copy of stored message content.
+
+    Older multimodal user messages may be persisted as a JSON *string*
+    containing image_url blocks with inline base64 image bytes. Those bytes are
+    needed for model calls when the turn is first sent, but they should not be
+    sent back through /api/history every time the user opens the chat. The
+    attachment metadata already carries file ids/names for the UI cards.
+    """
+    if isinstance(content, list):
+        text_parts = []
+        omitted_media = 0
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "text":
+                text = block.get("text")
+                if isinstance(text, str) and text:
+                    text_parts.append(text)
+            elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
+                omitted_media += 1
+        text = "\n".join(text_parts).strip()
+        if omitted_media and not text:
+            return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
+        return text
+
+    if not isinstance(content, str):
+        return content
+    if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
+        return content
+
+    stripped = content.lstrip()
+    if stripped.startswith("["):
+        try:
+            blocks = json.loads(content)
+        except (json.JSONDecodeError, TypeError, ValueError):
+            blocks = None
+        if isinstance(blocks, list):
+            text_parts = []
+            for block in blocks:
+                if not isinstance(block, dict):
+                    continue
+                if block.get("type") == "text":
+                    text = block.get("text")
+                    if isinstance(text, str) and text:
+                        text_parts.append(text)
+            if text_parts:
+                return "\n".join(text_parts).strip()
+
+    if "data:image/" in content:
+        return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
+    return content
+
+
+def _merge_continue_rows_to_delete(db_messages, db1, db2):
+    """DB rows to delete when merging the last two assistant messages.
+
+    Always the second assistant message (db2), plus ONLY the single
+    intervening "continue" user message (the one carrying "previous response
+    was interrupted") — matching the in-memory merge. The previous code
+    deleted the whole index range between the two assistant rows, destroying
+    any tool/system/user messages in between and desyncing the DB from the
+    in-memory history.
+    """
+    to_delete = [db2]
+    i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
+    i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
+    if i1 is not None and i2 is not None and i2 - 1 > i1:
+        between = db_messages[i2 - 1]
+        if getattr(between, "role", "") == "user" and            "previous response was interrupted" in (getattr(between, "content", "") or ""):
+            to_delete.append(between)
+    return to_delete
+
+
+def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
+    router = APIRouter(tags=["history"])
+
+    def _reserve_message_uploads(
+        request: Request,
+        content: Any,
+        metadata: Any = None,
+    ) -> None:
+        try:
+            missing_id = reserve_message_upload_references(
+                upload_handler,
+                effective_user(request),
+                content,
+                metadata,
+            )
+        except (TypeError, ValueError) as exc:
+            raise HTTPException(400, "Invalid message attachment metadata") from exc
+        if missing_id:
+            raise HTTPException(
+                409,
+                f"Referenced upload is no longer available: {missing_id}",
+            )
+
+    def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
+        entry = {"role": m.role, "content": _history_display_content(m.content)}
+        meta = {}
+        if m.meta_data:
+            try:
+                meta = json.loads(m.meta_data) or {}
+            except (json.JSONDecodeError, ValueError):
+                meta = {}
+        if m.timestamp and "timestamp" not in meta:
+            meta["timestamp"] = m.timestamp.isoformat() + "Z"
+        if meta:
+            entry["metadata"] = meta
+        return entry
+
+    @router.get("/api/history/{session_id}")
+    async def get_session_history(
+        request: Request,
+        session_id: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Dict[str, Any]:
+        _verify_session_owner(request, session_id)
+        if limit is not None:
+            page_limit = max(1, min(int(limit), 100))
+            db = SessionLocal()
+            try:
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session is None:
+                    raise HTTPException(404, f"Session '{session_id}' not found")
+
+                total = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .count()
+                )
+                page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
+                page_offset = max(0, min(page_offset, total))
+                rows = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .offset(page_offset)
+                    .limit(page_limit)
+                    .all()
+                )
+                history_dict = [
+                    entry for entry in (_db_history_entry(m) for m in rows)
+                    if not (entry.get("metadata") or {}).get("hidden")
+                ]
+                return {
+                    "history": history_dict,
+                    "model": db_session.model,
+                    "endpoint_url": db_session.endpoint_url,
+                    "name": db_session.name,
+                    "offset": page_offset,
+                    "limit": page_limit,
+                    "total": total,
+                    "has_more_before": page_offset > 0,
+                    "has_more_after": page_offset + len(rows) < total,
+                }
+            finally:
+                db.close()
+
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, f"Session '{session_id}' not found")
+
+        history_dict = []
+        for msg in session.history:
+            if isinstance(msg, ChatMessage):
+                # Skip hidden messages (e.g. compaction summaries for AI context)
+                if msg.metadata and msg.metadata.get("hidden"):
+                    continue
+                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
+                if msg.metadata:
+                    entry["metadata"] = msg.metadata
+                history_dict.append(entry)
+            elif isinstance(msg, dict):
+                if msg.get("metadata", {}).get("hidden"):
+                    continue
+                entry = {
+                    "role": msg.get("role", ""),
+                    "content": _history_display_content(msg.get("content", "")),
+                }
+                if msg.get("metadata"):
+                    entry["metadata"] = msg["metadata"]
+                history_dict.append(entry)
+
+        # Fallback: load from DB if in-memory is empty
+        if not history_dict:
+            db = SessionLocal()
+            try:
+                db_messages = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .all()
+                )
+                db_history = []
+                for m in db_messages:
+                    db_history.append(_db_history_entry(m))
+                if db_history:
+                    # Rebuild in-memory history from the full set so hidden
+                    # messages (e.g. compaction summaries) are kept for AI context.
+                    session.history = [
+                        ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
+                        for m in db_history
+                    ]
+                # Response excludes hidden messages, matching the in-memory path.
+                history_dict = [
+                    m for m in db_history
+                    if not (m.get("metadata") or {}).get("hidden")
+                ]
+            except Exception as e:
+                logger.error(f"DB fallback failed for {session_id}: {e}")
+            finally:
+                db.close()
+
+        return {
+            "history": history_dict,
+            "model": session.model,
+            "endpoint_url": session.endpoint_url,
+            "name": session.name,
+        }
+
+    @router.post("/api/session/{session_id}/truncate")
+    async def truncate_session(request: Request, session_id: str):
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            keep_count = body.get("keep_count", 0)
+            result = session_manager.truncate_messages(session_id, keep_count)
+            return {"status": "ok", "kept": keep_count, "truncated": result}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Truncate error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/message")
+    async def add_message(request: Request, session_id: str):
+        """Add a message to a session (for slash command persistence)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            role = body.get("role", "assistant")
+            content = body.get("content", "")
+            if not content:
+                raise HTTPException(400, "content is required")
+            metadata = body.get("metadata")
+            _reserve_message_uploads(request, content, metadata)
+            msg = ChatMessage(role=role, content=content, metadata=metadata)
+            session_manager.add_message(session_id, msg)
+            return {"status": "ok"}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+    @router.post("/api/session/{session_id}/delete-messages")
+    async def delete_messages(request: Request, session_id: str):
+        """Delete specific messages by DB ID (or legacy index)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            msg_ids = body.get("msg_ids", [])
+            indices = body.get("indices")  # legacy fallback
+
+            session = session_manager.get_session(session_id)
+            db = SessionLocal()
+            try:
+                if msg_ids:
+                    # New ID-based delete
+                    deleted = 0
+                    for mid in msg_ids:
+                        db_msg = db.query(DbChatMessage).filter(
+                            DbChatMessage.id == mid,
+                            DbChatMessage.session_id == session_id,
+                        ).first()
+                        if db_msg:
+                            db.delete(db_msg)
+                            deleted += 1
+
+                    # Remove from in-memory history by matching _db_id
+                    def _get_db_id(m):
+                        meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
+                        return meta.get('_db_id') if isinstance(meta, dict) else None
+                    session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
+                elif indices:
+                    # Legacy index-based delete
+                    indices = sorted(indices, reverse=True)
+                    db_messages = db.query(DbChatMessage).filter(
+                        DbChatMessage.session_id == session_id
+                    ).order_by(DbChatMessage.timestamp).all()
+
+                    deleted = 0
+                    for idx in indices:
+                        if 0 <= idx < len(db_messages):
+                            db.delete(db_messages[idx])
+                            deleted += 1
+                        if 0 <= idx < len(session.history):
+                            session.history.pop(idx)
+                else:
+                    return {"status": "ok", "deleted": 0}
+
+                session.message_count = len(session.history)
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session:
+                    db_session.message_count = len(session.history)
+                    from datetime import datetime, timezone
+                    db_session.updated_at = datetime.now(timezone.utc)
+
+                db.commit()
+                return {"status": "ok", "deleted": deleted}
+            finally:
+                db.close()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Delete messages error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/edit-message")
+    async def edit_message(request: Request, session_id: str):
+        """Edit the content of a message by its database ID."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            msg_id = body.get("msg_id")
+            content = body.get("content")
+            if not msg_id or content is None:
+                raise HTTPException(400, "msg_id and content are required")
+
+            _reserve_message_uploads(request, content)
+
+            session = session_manager.get_session(session_id)
+            db = SessionLocal()
+            try:
+                db_msg = db.query(DbChatMessage).filter(
+                    DbChatMessage.id == msg_id,
+                    DbChatMessage.session_id == session_id,
+                ).first()
+                if not db_msg:
+                    raise HTTPException(404, "Message not found")
+
+                db_msg.content = content
+                meta = {}
+                if db_msg.meta_data:
+                    try: meta = json.loads(db_msg.meta_data)
+                    except (json.JSONDecodeError, ValueError): pass
+                meta['edited'] = True
+                db_msg.meta_data = json.dumps(meta)
+
+                # Update in-memory history by matching _db_id
+                for hmsg in session.history:
+                    hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
+                    if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
+                        if isinstance(hmsg, ChatMessage):
+                            hmsg.content = content
+                            hmsg.metadata['edited'] = True
+                        elif isinstance(hmsg, dict):
+                            hmsg['content'] = content
+                            hmsg['metadata']['edited'] = True
+                        break
+
+                db.commit()
+                return {"status": "ok"}
+            finally:
+                db.close()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Edit message error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/mark-stopped")
+    async def mark_stopped(request: Request, session_id: str):
+        """Mark the last assistant message as stopped by user."""
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+            # Find last assistant message and add stopped metadata
+            for msg in reversed(session.history):
+                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
+                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
+                    if isinstance(msg, ChatMessage):
+                        if not msg.metadata:
+                            msg.metadata = {}
+                        msg.metadata['stopped'] = True
+                        if not msg.metadata.get('model'):
+                            msg.metadata['model'] = session.model
+                    else:
+                        if 'metadata' not in msg:
+                            msg['metadata'] = {}
+                        msg['metadata']['stopped'] = True
+                        if not msg['metadata'].get('model'):
+                            msg['metadata']['model'] = session.model
+                    break
+            # Also update in DB
+            db = SessionLocal()
+            try:
+                import json as _json
+                db_messages = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
+                    .order_by(DbChatMessage.timestamp.desc())
+                    .first()
+                )
+                if db_messages:
+                    meta = {}
+                    if db_messages.meta_data:
+                        try:
+                            meta = _json.loads(db_messages.meta_data)
+                        except (json.JSONDecodeError, ValueError):
+                            pass
+                    meta['stopped'] = True
+                    if not meta.get('model'):
+                        meta['model'] = session.model
+                    db_messages.meta_data = _json.dumps(meta)
+                    db.commit()
+            finally:
+                db.close()
+            session_manager.save_sessions()
+            return {"status": "ok"}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Mark stopped error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/update-last-meta")
+    async def update_last_meta(request: Request, session_id: str):
+        """Merge metadata into the last assistant message (e.g. save variants)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            meta_update = body.get("metadata", {})
+            session = session_manager.get_session(session_id)
+
+            # Update in-memory
+            for msg in reversed(session.history):
+                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
+                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
+                    if isinstance(msg, ChatMessage):
+                        if not msg.metadata:
+                            msg.metadata = {}
+                        msg.metadata.update(meta_update)
+                    else:
+                        if 'metadata' not in msg:
+                            msg['metadata'] = {}
+                        msg['metadata'].update(meta_update)
+                    break
+
+            # Update in DB
+            db = SessionLocal()
+            try:
+                import json as _json
+                db_msg = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
+                    .order_by(DbChatMessage.timestamp.desc())
+                    .first()
+                )
+                if db_msg:
+                    meta = {}
+                    if db_msg.meta_data:
+                        try: meta = _json.loads(db_msg.meta_data)
+                        except (json.JSONDecodeError, ValueError): pass
+                    meta.update(meta_update)
+                    db_msg.meta_data = _json.dumps(meta)
+                    db.commit()
+            finally:
+                db.close()
+            session_manager.save_sessions()
+            return {"status": "ok"}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Update last meta error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/merge-last-assistant")
+    async def merge_last_assistant(request: Request, session_id: str):
+        """Merge the last two assistant messages into one (for continue)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            separator = body.get("separator", "\n\n")
+            session = session_manager.get_session(session_id)
+
+            # Find last two assistant messages in-memory
+            ai_indices = []
+            for i, msg in enumerate(session.history):
+                role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
+                if role == 'assistant':
+                    ai_indices.append(i)
+
+            if len(ai_indices) < 2:
+                return {"status": "ok", "merged": False}
+
+            idx1, idx2 = ai_indices[-2], ai_indices[-1]
+            msg1, msg2 = session.history[idx1], session.history[idx2]
+
+            content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
+            content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
+            merged_content = content1 + separator + content2
+
+            # Merge metadata
+            meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
+            meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
+            merged_meta = {**meta1, **meta2}
+            merged_meta.pop('stopped', None)  # no longer stopped after continue
+
+            # Update first message, remove second
+            if isinstance(msg1, ChatMessage):
+                msg1.content = merged_content
+                msg1.metadata = merged_meta
+            else:
+                msg1['content'] = merged_content
+                msg1['metadata'] = merged_meta
+
+            # Also remove the hidden "continue" user message between them if present
+            # It's the message at idx2-1 if it's a user message with continue text
+            remove_indices = [idx2]
+            if idx2 - 1 > idx1:
+                between = session.history[idx2 - 1]
+                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
+                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
+                if between_role == 'user' and 'previous response was interrupted' in between_content:
+                    remove_indices.insert(0, idx2 - 1)
+
+            for ri in sorted(remove_indices, reverse=True):
+                session.history.pop(ri)
+
+            # Update DB
+            db = SessionLocal()
+            try:
+                import json as _json
+                db_messages = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .all()
+                )
+                # Find last two assistant messages in DB
+                ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
+                if len(ai_db) >= 2:
+                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
+                    db1.content = merged_content
+                    db1.meta_data = _json.dumps(merged_meta)
+
+                    # Mirror the in-memory deletion: remove the second assistant
+                    # message and ONLY the "continue" user message between them
+                    # (not arbitrary tool/system/user rows). The old
+                    # range-delete destroyed every row between the two assistant
+                    # messages, desyncing the DB from the in-memory history.
+                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
+                        db.delete(_row)
+
+                    db.commit()
+            finally:
+                db.close()
+            session_manager.save_sessions()
+            return {"status": "ok", "merged": True}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Merge assistant error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/fork")
+    async def fork_session(request: Request, session_id: str):
+        """Create a new session with messages copied up to keep_count."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            keep_count = body.get("keep_count", 0)
+
+            # Get the source session
+            source = session_manager.sessions.get(session_id)
+            if not source:
+                raise HTTPException(404, "Session not found")
+
+            # Create new session
+            new_id = str(uuid.uuid4())
+            fork_name = f"\u2ADD {source.name}"
+            new_session = session_manager.create_session(
+                session_id=new_id,
+                name=fork_name,
+                endpoint_url=source.endpoint_url,
+                model=source.model,
+                rag=False,
+                owner=getattr(source, 'owner', None),
+            )
+
+            # Copy messages up to keep_count
+            msgs_to_copy = source.history[:keep_count]
+            for msg in msgs_to_copy:
+                # Copy the metadata dict. Sharing it would let the fork's
+                # persistence (add_message -> _persist_message stamps
+                # _db_id/timestamp onto the dict) mutate the SOURCE session's
+                # in-memory messages, corrupting their _db_id and breaking
+                # edit/delete-by-id on the original conversation.
+                meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
+                new_session.add_message(ChatMessage(msg.role, msg.content, meta))
+            try:
+                from src.event_bus import fire_event
+                fire_event("session_created", getattr(source, 'owner', None))
+            except Exception:
+                logger.debug("session_created event dispatch failed", exc_info=True)
+
+            return {
+                "status": "ok",
+                "id": new_id,
+                "name": fork_name,
+                "kept": len(msgs_to_copy),
+            }
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Fork error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.get("/api/conversations/topics")
+    async def get_conversation_topics(request: Request) -> Dict[str, Any]:
+        from src.auth_helpers import require_user
+        user = require_user(request)
+        try:
+            return analyze_topics(session_manager, owner=user or None)
+        except Exception as e:
+            raise HTTPException(500, f"Topic analysis failed: {e}")
+
+    @router.post("/api/session/{session_id}/compact")
+    async def compact_session(request: Request, session_id: str):
+        """Manually trigger context compaction for a session."""
+        _verify_session_owner(request, session_id)
+        from src.auth_helpers import effective_user
+        owner = effective_user(request)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        _reject_compact_during_active_run(session_id)
+
+        try:
+            from src.model_context import estimate_tokens, get_context_length
+            from src.llm_core import llm_call_async
+            from src.endpoint_resolver import resolve_endpoint
+
+            if len(session.history) < 6:
+                return {"status": "ok", "message": "Not enough messages to compact"}
+
+            ctx_len = get_context_length(session.endpoint_url, session.model)
+            messages_before = session.get_context_messages()
+            used_before = estimate_tokens(messages_before)
+            pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
+            msg_count_before = len(session.history)
+
+            # Keep only last 4 messages, summarize the rest
+            keep_count = 4
+            older = session.history[:-keep_count]
+            recent = session.history[-keep_count:]
+
+            # Build text to summarize
+            convo_text = "\n".join(
+                f"{_message_role(m).upper()}: "
+                f"{_message_text(m)[:2000]}"
+                for m in older
+            )
+
+            # Use utility model if available
+            util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
+            compact_url = util_url or session.endpoint_url
+            compact_model = util_model or session.model
+            compact_headers = util_headers if util_url else session.headers
+
+            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
+            compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
+            sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
+            summary = await llm_call_async(
+                compact_url, compact_model,
+                [
+                    {"role": "system", "content": sys_prompt},
+                    {"role": "user", "content": convo_text},
+                ],
+                temperature=0.2, max_tokens=1024,
+                headers=compact_headers, timeout=30,
+            )
+
+            # Replace session history: summary as system message + recent messages
+            # System message holds the full summary for AI context
+            system_summary = ChatMessage(
+                role="system",
+                content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
+                metadata={"compacted": True, "hidden": True},
+            )
+            # Visible assistant message just shows stats
+            summary_msg = ChatMessage(
+                role="assistant",
+                content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
+                metadata={"compacted": True, "messages_removed": len(older)},
+            )
+            new_history = [system_summary, summary_msg] + list(recent)
+            session.history = new_history
+            session.message_count = len(session.history)
+            logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
+
+            # Update DB: delete old messages, insert summary
+            db = SessionLocal()
+            try:
+                db_msgs = db.query(DbChatMessage).filter(
+                    DbChatMessage.session_id == session_id
+                ).order_by(DbChatMessage.timestamp).all()
+
+                # Delete all but the last keep_count
+                for m in db_msgs[:-keep_count]:
+                    db.delete(m)
+
+                # Insert system summary (hidden, for AI context) and visible summary
+                import json as _json
+                import uuid
+                from datetime import datetime, timezone
+                now = datetime.now(timezone.utc)
+                db_sys_summary = DbChatMessage(
+                    id=str(uuid.uuid4()),
+                    session_id=session_id,
+                    role="system",
+                    content=system_summary.content,
+                    meta_data=_json.dumps(system_summary.metadata),
+                    timestamp=now,
+                )
+                db.add(db_sys_summary)
+                db_summary = DbChatMessage(
+                    id=str(uuid.uuid4()),
+                    session_id=session_id,
+                    role="assistant",
+                    content=summary_msg.content,
+                    meta_data=_json.dumps(summary_msg.metadata),
+                    timestamp=now,
+                )
+                db.add(db_summary)
+
+                # Update session record
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session:
+                    db_session.message_count = len(session.history)
+                    db_session.updated_at = datetime.now(timezone.utc)
+                db.commit()
+            finally:
+                db.close()
+
+            session_manager.save_sessions()
+
+            used_after = estimate_tokens(session.get_context_messages())
+            pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
+
+            return {
+                "status": "ok",
+                "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
+                "before": pct_before,
+                "after": pct_after,
+            }
+
+        except Exception as e:
+            logger.error(f"Manual compact error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    return router

routes/history_routes.py

@@ -1,768 +1,17 @@
-"""History routes — session history, truncation, fork, conversation topics."""
+"""Backward-compat shim — canonical location is routes/history/history_routes.py.
 
-import json
+This module is replaced in ``sys.modules`` by the canonical module object so
-import uuid
+that ``import routes.history_routes``, ``from routes.history_routes import X``,
-import logging
+``importlib.import_module("routes.history_routes")``, and the
-import re
+``import ... as history_routes`` + ``monkeypatch.setattr(history_routes, ...)``
-from typing import Dict, Any, Optional
+pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py
+all operate on the *same* object the application actually uses. Keeps existing
+import paths working after slice 2d (#4082/#4071). Source-introspection tests
+read the canonical file by path.
+"""
 
-from fastapi import APIRouter, Request, HTTPException
+import sys as _sys
 
-from core.models import ChatMessage
+from routes.history import history_routes as _canonical  # noqa: F401
-from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
-from src.topic_analyzer import analyze_topics
-from routes.session_routes import (
-    _message_role,
-    _message_text,
-    _reject_compact_during_active_run,
-    _verify_session_owner,
-)
 
-logger = logging.getLogger(__name__)
+_sys.modules[__name__] = _canonical
-
-_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
-_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
-
-
-def _history_display_content(content: Any) -> Any:
-    """Return a lightweight browser-display copy of stored message content.
-
-    Older multimodal user messages may be persisted as a JSON *string*
-    containing image_url blocks with inline base64 image bytes. Those bytes are
-    needed for model calls when the turn is first sent, but they should not be
-    sent back through /api/history every time the user opens the chat. The
-    attachment metadata already carries file ids/names for the UI cards.
-    """
-    if isinstance(content, list):
-        text_parts = []
-        omitted_media = 0
-        for block in content:
-            if not isinstance(block, dict):
-                continue
-            if block.get("type") == "text":
-                text = block.get("text")
-                if isinstance(text, str) and text:
-                    text_parts.append(text)
-            elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
-                omitted_media += 1
-        text = "\n".join(text_parts).strip()
-        if omitted_media and not text:
-            return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
-        return text
-
-    if not isinstance(content, str):
-        return content
-    if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
-        return content
-
-    stripped = content.lstrip()
-    if stripped.startswith("["):
-        try:
-            blocks = json.loads(content)
-        except (json.JSONDecodeError, TypeError, ValueError):
-            blocks = None
-        if isinstance(blocks, list):
-            text_parts = []
-            for block in blocks:
-                if not isinstance(block, dict):
-                    continue
-                if block.get("type") == "text":
-                    text = block.get("text")
-                    if isinstance(text, str) and text:
-                        text_parts.append(text)
-            if text_parts:
-                return "\n".join(text_parts).strip()
-
-    if "data:image/" in content:
-        return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
-    return content
-
-
-def _merge_continue_rows_to_delete(db_messages, db1, db2):
-    """DB rows to delete when merging the last two assistant messages.
-
-    Always the second assistant message (db2), plus ONLY the single
-    intervening "continue" user message (the one carrying "previous response
-    was interrupted") — matching the in-memory merge. The previous code
-    deleted the whole index range between the two assistant rows, destroying
-    any tool/system/user messages in between and desyncing the DB from the
-    in-memory history.
-    """
-    to_delete = [db2]
-    i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
-    i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
-    if i1 is not None and i2 is not None and i2 - 1 > i1:
-        between = db_messages[i2 - 1]
-        if getattr(between, "role", "") == "user" and            "previous response was interrupted" in (getattr(between, "content", "") or ""):
-            to_delete.append(between)
-    return to_delete
-
-
-def setup_history_routes(session_manager) -> APIRouter:
-    router = APIRouter(tags=["history"])
-
-    def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
-        entry = {"role": m.role, "content": _history_display_content(m.content)}
-        meta = {}
-        if m.meta_data:
-            try:
-                meta = json.loads(m.meta_data) or {}
-            except (json.JSONDecodeError, ValueError):
-                meta = {}
-        if m.timestamp and "timestamp" not in meta:
-            meta["timestamp"] = m.timestamp.isoformat() + "Z"
-        if meta:
-            entry["metadata"] = meta
-        return entry
-
-    @router.get("/api/history/{session_id}")
-    async def get_session_history(
-        request: Request,
-        session_id: str,
-        limit: Optional[int] = None,
-        offset: Optional[int] = None,
-    ) -> Dict[str, Any]:
-        _verify_session_owner(request, session_id)
-        if limit is not None:
-            page_limit = max(1, min(int(limit), 100))
-            db = SessionLocal()
-            try:
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session is None:
-                    raise HTTPException(404, f"Session '{session_id}' not found")
-
-                total = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .count()
-                )
-                page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
-                page_offset = max(0, min(page_offset, total))
-                rows = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .offset(page_offset)
-                    .limit(page_limit)
-                    .all()
-                )
-                history_dict = [
-                    entry for entry in (_db_history_entry(m) for m in rows)
-                    if not (entry.get("metadata") or {}).get("hidden")
-                ]
-                return {
-                    "history": history_dict,
-                    "model": db_session.model,
-                    "endpoint_url": db_session.endpoint_url,
-                    "name": db_session.name,
-                    "offset": page_offset,
-                    "limit": page_limit,
-                    "total": total,
-                    "has_more_before": page_offset > 0,
-                    "has_more_after": page_offset + len(rows) < total,
-                }
-            finally:
-                db.close()
-
-        try:
-            session = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session '{session_id}' not found")
-
-        history_dict = []
-        for msg in session.history:
-            if isinstance(msg, ChatMessage):
-                # Skip hidden messages (e.g. compaction summaries for AI context)
-                if msg.metadata and msg.metadata.get("hidden"):
-                    continue
-                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
-                if msg.metadata:
-                    entry["metadata"] = msg.metadata
-                history_dict.append(entry)
-            elif isinstance(msg, dict):
-                if msg.get("metadata", {}).get("hidden"):
-                    continue
-                entry = {
-                    "role": msg.get("role", ""),
-                    "content": _history_display_content(msg.get("content", "")),
-                }
-                if msg.get("metadata"):
-                    entry["metadata"] = msg["metadata"]
-                history_dict.append(entry)
-
-        # Fallback: load from DB if in-memory is empty
-        if not history_dict:
-            db = SessionLocal()
-            try:
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .all()
-                )
-                db_history = []
-                for m in db_messages:
-                    db_history.append(_db_history_entry(m))
-                if db_history:
-                    # Rebuild in-memory history from the full set so hidden
-                    # messages (e.g. compaction summaries) are kept for AI context.
-                    session.history = [
-                        ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
-                        for m in db_history
-                    ]
-                # Response excludes hidden messages, matching the in-memory path.
-                history_dict = [
-                    m for m in db_history
-                    if not (m.get("metadata") or {}).get("hidden")
-                ]
-            except Exception as e:
-                logger.error(f"DB fallback failed for {session_id}: {e}")
-            finally:
-                db.close()
-
-        return {
-            "history": history_dict,
-            "model": session.model,
-            "endpoint_url": session.endpoint_url,
-            "name": session.name,
-        }
-
-    @router.post("/api/session/{session_id}/truncate")
-    async def truncate_session(request: Request, session_id: str):
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            keep_count = body.get("keep_count", 0)
-            result = session_manager.truncate_messages(session_id, keep_count)
-            return {"status": "ok", "kept": keep_count, "truncated": result}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Truncate error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/message")
-    async def add_message(request: Request, session_id: str):
-        """Add a message to a session (for slash command persistence)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            role = body.get("role", "assistant")
-            content = body.get("content", "")
-            if not content:
-                raise HTTPException(400, "content is required")
-            msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
-            session_manager.add_message(session_id, msg)
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-
-    @router.post("/api/session/{session_id}/delete-messages")
-    async def delete_messages(request: Request, session_id: str):
-        """Delete specific messages by DB ID (or legacy index)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            msg_ids = body.get("msg_ids", [])
-            indices = body.get("indices")  # legacy fallback
-
-            session = session_manager.get_session(session_id)
-            db = SessionLocal()
-            try:
-                if msg_ids:
-                    # New ID-based delete
-                    deleted = 0
-                    for mid in msg_ids:
-                        db_msg = db.query(DbChatMessage).filter(
-                            DbChatMessage.id == mid,
-                            DbChatMessage.session_id == session_id,
-                        ).first()
-                        if db_msg:
-                            db.delete(db_msg)
-                            deleted += 1
-
-                    # Remove from in-memory history by matching _db_id
-                    def _get_db_id(m):
-                        meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
-                        return meta.get('_db_id') if isinstance(meta, dict) else None
-                    session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
-                elif indices:
-                    # Legacy index-based delete
-                    indices = sorted(indices, reverse=True)
-                    db_messages = db.query(DbChatMessage).filter(
-                        DbChatMessage.session_id == session_id
-                    ).order_by(DbChatMessage.timestamp).all()
-
-                    deleted = 0
-                    for idx in indices:
-                        if 0 <= idx < len(db_messages):
-                            db.delete(db_messages[idx])
-                            deleted += 1
-                        if 0 <= idx < len(session.history):
-                            session.history.pop(idx)
-                else:
-                    return {"status": "ok", "deleted": 0}
-
-                session.message_count = len(session.history)
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session:
-                    db_session.message_count = len(session.history)
-                    from datetime import datetime, timezone
-                    db_session.updated_at = datetime.now(timezone.utc)
-
-                db.commit()
-                return {"status": "ok", "deleted": deleted}
-            finally:
-                db.close()
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Delete messages error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/edit-message")
-    async def edit_message(request: Request, session_id: str):
-        """Edit the content of a message by its database ID."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            msg_id = body.get("msg_id")
-            content = body.get("content")
-            if not msg_id or content is None:
-                raise HTTPException(400, "msg_id and content are required")
-
-            session = session_manager.get_session(session_id)
-            db = SessionLocal()
-            try:
-                db_msg = db.query(DbChatMessage).filter(
-                    DbChatMessage.id == msg_id,
-                    DbChatMessage.session_id == session_id,
-                ).first()
-                if not db_msg:
-                    raise HTTPException(404, "Message not found")
-
-                db_msg.content = content
-                meta = {}
-                if db_msg.meta_data:
-                    try: meta = json.loads(db_msg.meta_data)
-                    except (json.JSONDecodeError, ValueError): pass
-                meta['edited'] = True
-                db_msg.meta_data = json.dumps(meta)
-
-                # Update in-memory history by matching _db_id
-                for hmsg in session.history:
-                    hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
-                    if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
-                        if isinstance(hmsg, ChatMessage):
-                            hmsg.content = content
-                            hmsg.metadata['edited'] = True
-                        elif isinstance(hmsg, dict):
-                            hmsg['content'] = content
-                            hmsg['metadata']['edited'] = True
-                        break
-
-                db.commit()
-                return {"status": "ok"}
-            finally:
-                db.close()
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except HTTPException:
-            raise
-        except Exception as e:
-            logger.error(f"Edit message error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/mark-stopped")
-    async def mark_stopped(request: Request, session_id: str):
-        """Mark the last assistant message as stopped by user."""
-        _verify_session_owner(request, session_id)
-        try:
-            session = session_manager.get_session(session_id)
-            # Find last assistant message and add stopped metadata
-            for msg in reversed(session.history):
-                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
-                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
-                    if isinstance(msg, ChatMessage):
-                        if not msg.metadata:
-                            msg.metadata = {}
-                        msg.metadata['stopped'] = True
-                        if not msg.metadata.get('model'):
-                            msg.metadata['model'] = session.model
-                    else:
-                        if 'metadata' not in msg:
-                            msg['metadata'] = {}
-                        msg['metadata']['stopped'] = True
-                        if not msg['metadata'].get('model'):
-                            msg['metadata']['model'] = session.model
-                    break
-            # Also update in DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
-                    .order_by(DbChatMessage.timestamp.desc())
-                    .first()
-                )
-                if db_messages:
-                    meta = {}
-                    if db_messages.meta_data:
-                        try:
-                            meta = _json.loads(db_messages.meta_data)
-                        except (json.JSONDecodeError, ValueError):
-                            pass
-                    meta['stopped'] = True
-                    if not meta.get('model'):
-                        meta['model'] = session.model
-                    db_messages.meta_data = _json.dumps(meta)
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Mark stopped error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/update-last-meta")
-    async def update_last_meta(request: Request, session_id: str):
-        """Merge metadata into the last assistant message (e.g. save variants)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            meta_update = body.get("metadata", {})
-            session = session_manager.get_session(session_id)
-
-            # Update in-memory
-            for msg in reversed(session.history):
-                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
-                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
-                    if isinstance(msg, ChatMessage):
-                        if not msg.metadata:
-                            msg.metadata = {}
-                        msg.metadata.update(meta_update)
-                    else:
-                        if 'metadata' not in msg:
-                            msg['metadata'] = {}
-                        msg['metadata'].update(meta_update)
-                    break
-
-            # Update in DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_msg = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
-                    .order_by(DbChatMessage.timestamp.desc())
-                    .first()
-                )
-                if db_msg:
-                    meta = {}
-                    if db_msg.meta_data:
-                        try: meta = _json.loads(db_msg.meta_data)
-                        except (json.JSONDecodeError, ValueError): pass
-                    meta.update(meta_update)
-                    db_msg.meta_data = _json.dumps(meta)
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Update last meta error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/merge-last-assistant")
-    async def merge_last_assistant(request: Request, session_id: str):
-        """Merge the last two assistant messages into one (for continue)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            separator = body.get("separator", "\n\n")
-            session = session_manager.get_session(session_id)
-
-            # Find last two assistant messages in-memory
-            ai_indices = []
-            for i, msg in enumerate(session.history):
-                role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
-                if role == 'assistant':
-                    ai_indices.append(i)
-
-            if len(ai_indices) < 2:
-                return {"status": "ok", "merged": False}
-
-            idx1, idx2 = ai_indices[-2], ai_indices[-1]
-            msg1, msg2 = session.history[idx1], session.history[idx2]
-
-            content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
-            content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
-            merged_content = content1 + separator + content2
-
-            # Merge metadata
-            meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
-            meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
-            merged_meta = {**meta1, **meta2}
-            merged_meta.pop('stopped', None)  # no longer stopped after continue
-
-            # Update first message, remove second
-            if isinstance(msg1, ChatMessage):
-                msg1.content = merged_content
-                msg1.metadata = merged_meta
-            else:
-                msg1['content'] = merged_content
-                msg1['metadata'] = merged_meta
-
-            # Also remove the hidden "continue" user message between them if present
-            # It's the message at idx2-1 if it's a user message with continue text
-            remove_indices = [idx2]
-            if idx2 - 1 > idx1:
-                between = session.history[idx2 - 1]
-                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
-                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
-                if between_role == 'user' and 'previous response was interrupted' in between_content:
-                    remove_indices.insert(0, idx2 - 1)
-
-            for ri in sorted(remove_indices, reverse=True):
-                session.history.pop(ri)
-
-            # Update DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .all()
-                )
-                # Find last two assistant messages in DB
-                ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
-                if len(ai_db) >= 2:
-                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
-                    db1.content = merged_content
-                    db1.meta_data = _json.dumps(merged_meta)
-
-                    # Mirror the in-memory deletion: remove the second assistant
-                    # message and ONLY the "continue" user message between them
-                    # (not arbitrary tool/system/user rows). The old
-                    # range-delete destroyed every row between the two assistant
-                    # messages, desyncing the DB from the in-memory history.
-                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
-                        db.delete(_row)
-
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok", "merged": True}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Merge assistant error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/fork")
-    async def fork_session(request: Request, session_id: str):
-        """Create a new session with messages copied up to keep_count."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            keep_count = body.get("keep_count", 0)
-
-            # Get the source session
-            source = session_manager.sessions.get(session_id)
-            if not source:
-                raise HTTPException(404, "Session not found")
-
-            # Create new session
-            new_id = str(uuid.uuid4())
-            fork_name = f"\u2ADD {source.name}"
-            new_session = session_manager.create_session(
-                session_id=new_id,
-                name=fork_name,
-                endpoint_url=source.endpoint_url,
-                model=source.model,
-                rag=False,
-                owner=getattr(source, 'owner', None),
-            )
-
-            # Copy messages up to keep_count
-            msgs_to_copy = source.history[:keep_count]
-            for msg in msgs_to_copy:
-                # Copy the metadata dict. Sharing it would let the fork's
-                # persistence (add_message -> _persist_message stamps
-                # _db_id/timestamp onto the dict) mutate the SOURCE session's
-                # in-memory messages, corrupting their _db_id and breaking
-                # edit/delete-by-id on the original conversation.
-                meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
-                new_session.add_message(ChatMessage(msg.role, msg.content, meta))
-            try:
-                from src.event_bus import fire_event
-                fire_event("session_created", getattr(source, 'owner', None))
-            except Exception:
-                logger.debug("session_created event dispatch failed", exc_info=True)
-
-            return {
-                "status": "ok",
-                "id": new_id,
-                "name": fork_name,
-                "kept": len(msgs_to_copy),
-            }
-        except HTTPException:
-            raise
-        except Exception as e:
-            logger.error(f"Fork error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.get("/api/conversations/topics")
-    async def get_conversation_topics(request: Request) -> Dict[str, Any]:
-        from src.auth_helpers import require_user
-        user = require_user(request)
-        try:
-            return analyze_topics(session_manager, owner=user or None)
-        except Exception as e:
-            raise HTTPException(500, f"Topic analysis failed: {e}")
-
-    @router.post("/api/session/{session_id}/compact")
-    async def compact_session(request: Request, session_id: str):
-        """Manually trigger context compaction for a session."""
-        _verify_session_owner(request, session_id)
-        from src.auth_helpers import effective_user
-        owner = effective_user(request)
-        try:
-            session = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        _reject_compact_during_active_run(session_id)
-
-        try:
-            from src.model_context import estimate_tokens, get_context_length
-            from src.llm_core import llm_call_async
-            from src.endpoint_resolver import resolve_endpoint
-
-            if len(session.history) < 6:
-                return {"status": "ok", "message": "Not enough messages to compact"}
-
-            ctx_len = get_context_length(session.endpoint_url, session.model)
-            messages_before = session.get_context_messages()
-            used_before = estimate_tokens(messages_before)
-            pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
-            msg_count_before = len(session.history)
-
-            # Keep only last 4 messages, summarize the rest
-            keep_count = 4
-            older = session.history[:-keep_count]
-            recent = session.history[-keep_count:]
-
-            # Build text to summarize
-            convo_text = "\n".join(
-                f"{_message_role(m).upper()}: "
-                f"{_message_text(m)[:2000]}"
-                for m in older
-            )
-
-            # Use utility model if available
-            util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
-            compact_url = util_url or session.endpoint_url
-            compact_model = util_model or session.model
-            compact_headers = util_headers if util_url else session.headers
-
-            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
-            compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
-            sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
-            summary = await llm_call_async(
-                compact_url, compact_model,
-                [
-                    {"role": "system", "content": sys_prompt},
-                    {"role": "user", "content": convo_text},
-                ],
-                temperature=0.2, max_tokens=1024,
-                headers=compact_headers, timeout=30,
-            )
-
-            # Replace session history: summary as system message + recent messages
-            # System message holds the full summary for AI context
-            system_summary = ChatMessage(
-                role="system",
-                content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
-                metadata={"compacted": True, "hidden": True},
-            )
-            # Visible assistant message just shows stats
-            summary_msg = ChatMessage(
-                role="assistant",
-                content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
-                metadata={"compacted": True, "messages_removed": len(older)},
-            )
-            new_history = [system_summary, summary_msg] + list(recent)
-            session.history = new_history
-            session.message_count = len(session.history)
-            logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
-
-            # Update DB: delete old messages, insert summary
-            db = SessionLocal()
-            try:
-                db_msgs = db.query(DbChatMessage).filter(
-                    DbChatMessage.session_id == session_id
-                ).order_by(DbChatMessage.timestamp).all()
-
-                # Delete all but the last keep_count
-                for m in db_msgs[:-keep_count]:
-                    db.delete(m)
-
-                # Insert system summary (hidden, for AI context) and visible summary
-                import json as _json
-                import uuid
-                from datetime import datetime, timezone
-                now = datetime.now(timezone.utc)
-                db_sys_summary = DbChatMessage(
-                    id=str(uuid.uuid4()),
-                    session_id=session_id,
-                    role="system",
-                    content=system_summary.content,
-                    meta_data=_json.dumps(system_summary.metadata),
-                    timestamp=now,
-                )
-                db.add(db_sys_summary)
-                db_summary = DbChatMessage(
-                    id=str(uuid.uuid4()),
-                    session_id=session_id,
-                    role="assistant",
-                    content=summary_msg.content,
-                    meta_data=_json.dumps(summary_msg.metadata),
-                    timestamp=now,
-                )
-                db.add(db_summary)
-
-                # Update session record
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session:
-                    db_session.message_count = len(session.history)
-                    db_session.updated_at = datetime.now(timezone.utc)
-                db.commit()
-            finally:
-                db.close()
-
-            session_manager.save_sessions()
-
-            used_after = estimate_tokens(session.get_context_messages())
-            pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
-
-            return {
-                "status": "ok",
-                "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
-                "before": pct_before,
-                "after": pct_after,
-            }
-
-        except Exception as e:
-            logger.error(f"Manual compact error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    return router

routes/hwfit_routes.py

@@ -191,7 +191,7 @@ def setup_hwfit_routes():
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -200,11 +200,17 @@ def setup_hwfit_routes():
         fresh=true bypasses the hardware-detection cache."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.fit import rank_models
-        from services.hwfit.models import get_models, model_catalog_path
+        from services.hwfit.models import get_models, model_catalog_path, refresh_dynamic_catalogs
         host, ssh_port = _validate_detection_target(host, ssh_port)
         system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
         if system.get("error"):
             return {"system": system, "models": [], "error": system["error"]}
+        catalog_refresh = None
+        if refresh_catalog:
+            try:
+                catalog_refresh = refresh_dynamic_catalogs(force=True)
+            except Exception as e:
+                catalog_refresh = {"error": str(e)}
         if not get_models():
             return {
                 "system": system,
@@ -304,7 +310,10 @@ def setup_hwfit_routes():
             rank_kwargs.pop("target_context", None)
             rank_kwargs.pop("fit_only", None)
         results = rank_models(system, **rank_kwargs)
-        return {"system": system, "models": results}
+        payload = {"system": system, "models": results}
+        if catalog_refresh is not None:
+            payload["catalog_refresh"] = catalog_refresh
+        return payload
 
     @router.get("/profiles")
     def get_serve_profiles(model: str = "", model_path: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):

routes/model_routes.py

@@ -1152,6 +1152,36 @@ def _merge_model_ids(*lists):
     return out
 
 
+def _is_mlx_deepseek_v4_repo_id(model_id: str) -> bool:
+    m = str(model_id or "").lower()
+    return "mlx-community/deepseek-v4" in m
+
+
+def _is_mlx_deepseek_v4_shim_id(model_id: str) -> bool:
+    m = str(model_id or "").lower()
+    return "/.cache/odysseus/mlx-shims/deepseek-v4" in m
+
+
+def _filter_mlx_deepseek_v4_repo_when_shimmed(model_ids):
+    """Hide the broken MLX repo id when a launch-specific shim id is available.
+
+    mlx_lm.server may advertise the original HF repo id even though generation
+    only works through Odysseus' sanitized local shim. Keep the shim as the
+    submitted model id and remove the raw repo id from the picker/default list.
+    """
+    ids = list(model_ids or [])
+    has_shim = any(_is_mlx_deepseek_v4_shim_id(m) for m in ids)
+    if not has_shim:
+        return ids
+    return [m for m in ids if not _is_mlx_deepseek_v4_repo_id(m)]
+
+
+def _model_display_name(model_id: str) -> str:
+    if _is_mlx_deepseek_v4_shim_id(model_id):
+        return str(model_id or "").rstrip("/").split("/")[-1] or "DeepSeek-V4-Flash-4bit"
+    return str(model_id or "").split("/")[-1]
+
+
 def _visible_models(cached_models, hidden_models, pinned_models=None):
     """Merge cached + pinned model IDs, then filter out hidden ones.
 
@@ -1165,6 +1195,7 @@ def _visible_models(cached_models, hidden_models, pinned_models=None):
         _normalize_model_ids(cached_models),
         _normalize_model_ids(pinned_models),
     )
+    merged = _filter_mlx_deepseek_v4_repo_when_shimmed(merged)
     if not hidden_models:
         return merged
     hidden = set(_normalize_model_ids(hidden_models))
@@ -1396,9 +1427,9 @@ def setup_model_routes(model_discovery):
                     "port": 0,
                     "url": chat_url,
                     "models": curated,
-                    "models_display": [mid.split("/")[-1] for mid in curated],
+                    "models_display": [_model_display_name(mid) for mid in curated],
                     "models_extra": extra,
-                    "models_extra_display": [mid.split("/")[-1] for mid in extra],
+                    "models_extra_display": [_model_display_name(mid) for mid in extra],
                     "endpoint_id": ep.id,
                     "endpoint_name": ep.name,
                     "category": category,
@@ -1426,7 +1457,7 @@ def setup_model_routes(model_discovery):
         return {"hosts": [], "items": items}
 
     @router.get("/models")
-    def api_models(request: Request, refresh: bool = False, background: bool = True):
+    def api_models(request: Request, refresh: bool = False, background: bool = False):
         """Get available models — per-user (caller sees only their endpoints +
         legacy/shared null-owner rows). Cached per-user for 30s."""
         # Require auth; "" is the unconfigured single-user mode, treated as
@@ -1887,7 +1918,14 @@ def setup_model_routes(model_discovery):
                 if api_key.strip() and not existing.api_key:
                     existing.api_key = api_key.strip()
                     changed = True
-                if should_probe:
+                # Keep duplicate endpoint registration cheap. This path is hit
+                # by Cookbook/browser auto-register flows and can run while the
+                # user is sending a chat message. Probing a stale LAN endpoint
+                # here used to hold the request open for tens of seconds and
+                # contend with session creation, making "send" feel blocked.
+                # Explicit "require models" calls still probe; normal refresh
+                # belongs to /model-endpoints/{id}/models or /probe.
+                if require_model_list:
                     probed_models = _probe_endpoint(
                         base_url,
                         (api_key.strip() or existing.api_key or None),

routes/note_routes.py

@@ -13,6 +13,7 @@ from core.database import SessionLocal, Note
 from core.middleware import INTERNAL_TOOL_USER
 from src.auth_helpers import require_user
 from src.constants import DATA_DIR
+from src.upload_handler import reserve_upload_references
 from sqlalchemy.orm.attributes import flag_modified
 
 logger = logging.getLogger(__name__)
@@ -479,15 +480,28 @@ async def dispatch_reminder(
                 base = intg["base_url"].rstrip("/")
                 topic = settings.get("reminder_ntfy_topic") or "reminders"
                 ntfy_body = synthesis or note_body or title
-                hdrs = {"Title": title or "Reminder", "Priority": "high", "Tags": "bell"}
+                # ntfy Title is an ASCII HTTP header; sanitize Unicode and cap its length.
+                _clean_title = (title or "Reminder").encode("ascii", "replace").decode("ascii")[:200]
+                hdrs = {"Title": _clean_title, "Priority": "high", "Tags": "bell"}
                 api_key = intg.get("api_key", "")
                 if api_key:
                     hdrs["Authorization"] = f"Bearer {api_key}"
-                async with httpx.AsyncClient(timeout=10.0) as client:
+                # SSRF guard — same check (and env knob) as the webhook branch
-                    resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
+                # above: link-local / metadata addresses are always rejected;
-                    ntfy_sent = resp.is_success
+                # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
-                    if not ntfy_sent:
+                # so a ntfy base_url can't be pointed at internal services.
-                        ntfy_error = f"ntfy returned HTTP {resp.status_code}"
+                import os as _os
+                from src.url_safety import check_outbound_url as _chk
+                _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
+                _ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
+                if not _ok:
+                    ntfy_error = f"ntfy URL rejected: {_reason}"
+                else:
+                    async with httpx.AsyncClient(timeout=10.0) as client:
+                        resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
+                        ntfy_sent = resp.is_success
+                        if not ntfy_sent:
+                            ntfy_error = f"ntfy returned HTTP {resp.status_code}"
             else:
                 ntfy_error = "No enabled ntfy integration"
         except Exception as e:
@@ -561,7 +575,7 @@ async def dispatch_reminder(
 # Router factory
 # ---------------------------------------------------------------------------
 
-def setup_note_routes(task_scheduler=None):
+def setup_note_routes(task_scheduler=None, upload_handler=None):
     # Expose the scheduler to module-level `dispatch_reminder` so reminders
     # can also push to the in-app notification queue (the polling system
     # turns each entry into a real browser Notification + the existing
@@ -583,6 +597,11 @@ def setup_note_routes(task_scheduler=None):
         # did not.
         return require_user(request) or None
 
+    def _reserve_note_uploads(owner: Optional[str], *values) -> None:
+        missing_id = reserve_upload_references(upload_handler, owner, *values)
+        if missing_id:
+            raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
+
     def _is_admin_or_single_user(request: Request, user: str | None) -> bool:
         if user == INTERNAL_TOOL_USER:
             return True
@@ -632,6 +651,13 @@ def setup_note_routes(task_scheduler=None):
     @router.post("")
     def create_note(request: Request, body: NoteCreate):
         user = _owner(request)
+        _reserve_note_uploads(
+            user,
+            body.image_url,
+            body.color,
+            body.content,
+            json.dumps(body.items) if body.items is not None else None,
+        )
         db = SessionLocal()
         try:
             note = Note(
@@ -689,6 +715,13 @@ def setup_note_routes(task_scheduler=None):
             if user is not None and note.owner != user:
                 raise HTTPException(404, "Note not found")
 
+            _reserve_note_uploads(
+                user,
+                body.image_url,
+                body.color,
+                body.content,
+                json.dumps(body.items) if body.items is not None else None,
+            )
             if body.title is not None:
                 note.title = body.title
             if body.content is not None:

routes/research/research_routes.py

@@ -20,6 +20,55 @@ from src.constants import DEEP_RESEARCH_DIR
 
 _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
 
+
+def _validate_session_id(session_id: str) -> str:
+    if not _SESSION_ID_RE.fullmatch(session_id):
+        raise HTTPException(400, "Invalid session ID format")
+    return session_id
+
+
+def _research_storage_root() -> Path:
+    return Path(DEEP_RESEARCH_DIR).resolve()
+
+
+def _find_research_path(session_id: str) -> Path | None:
+    """Find a persisted research file without deriving its path from input."""
+    expected_name = f"{_validate_session_id(session_id)}.json"
+    root = _research_storage_root()
+    for stored_path in root.glob("*.json"):
+        if stored_path.name != expected_name:
+            continue
+        resolved = stored_path.resolve()
+        try:
+            resolved.relative_to(root)
+        except ValueError:
+            return None
+        if not resolved.is_file():
+            return None
+        return resolved
+    return None
+
+
+def _require_research_path(session_id: str) -> Path:
+    path = _find_research_path(session_id)
+    if path is None:
+        raise HTTPException(404, "Research not found")
+    return path
+
+
+def _find_owned_research_path(session_id: str, user: str) -> Path | None:
+    path = _find_research_path(session_id)
+    if path is None:
+        return None
+    try:
+        owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
+    except Exception:
+        return None
+    if owner != user:
+        return None
+    return path
+
+
 logger = logging.getLogger(__name__)
 
 # Model-name substrings that are NOT chat/generation models — research must
@@ -172,10 +221,6 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             raise HTTPException(401, "Not authenticated")
         return user
 
-    def _validate_session_id(session_id: str) -> None:
-        if not _SESSION_ID_RE.fullmatch(session_id):
-            raise HTTPException(400, "Invalid session ID format")
-
     def _owns_in_memory(session_id: str, user: str) -> bool:
         """Ownership check for an in-flight (in-memory) research task.
         Falls back to the on-disk JSON if the task has already finished."""
@@ -183,14 +228,34 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         if entry is not None:
             return entry.get("owner", "") == user
         # Task no longer in memory — check the persisted JSON.
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            return False
         try:
-            return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
+            return _find_owned_research_path(session_id, user) is not None
-        except Exception:
+        except HTTPException:
             return False
 
+    def _require_owned_or_active_research_path(session_id: str, user: str) -> Path | None:
+        """Validate ownership once and return the completed on-disk path.
+
+        Active running research has no completed disk path yet. Completed
+        tasks can remain in _active_tasks after persistence, so prefer their
+        owned disk path when available. Completed disk lookups still reuse the
+        path after the ownership gate.
+        """
+        entry = research_handler._active_tasks.get(session_id)
+        if entry is not None:
+            if entry.get("owner", "") != user:
+                raise HTTPException(404, "No research found for this session")
+            if entry.get("status") != "running":
+                path = _find_owned_research_path(session_id, user)
+                if path is not None:
+                    return path
+            return None
+
+        path = _find_owned_research_path(session_id, user)
+        if path is None:
+            raise HTTPException(404, "No research found for this session")
+        return path
+
     @router.get("/api/research/active")
     async def research_active(request: Request):
         """List all currently active (running) research tasks."""
@@ -247,9 +312,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
     def _assert_owns_research(session_id: str, user: str) -> None:
         """404-not-403 ownership gate for a research session's on-disk JSON.
         Use BEFORE returning any data or mutating the file."""
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        path = _require_research_path(session_id)
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
         try:
             owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
         except Exception:
@@ -361,9 +424,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         summary, stats — used by the Library preview panel."""
         user = _require_user(request)
         _validate_session_id(session_id)
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        path = _require_research_path(session_id)
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
         try:
             data = json.loads(path.read_text(encoding="utf-8"))
         except Exception as e:
@@ -378,9 +439,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         """Soft-archive / restore a research report (sets `archived` in its JSON)."""
         user = _require_user(request)
         _validate_session_id(session_id)
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        path = _require_research_path(session_id)
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
         try:
             data = json.loads(path.read_text(encoding="utf-8"))
             if data.get("owner") != user:
@@ -398,10 +457,9 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         """Delete a research result from disk."""
         user = _require_user(request)
         _validate_session_id(session_id)
-        data_dir = Path(DEEP_RESEARCH_DIR)
+        json_path = _find_research_path(session_id)
-        json_path = data_dir / f"{session_id}.json"
         deleted = False
-        if json_path.exists():
+        if json_path is not None:
             # SECURITY: verify ownership before letting the caller delete it.
             try:
                 data = json.loads(json_path.read_text(encoding="utf-8"))
@@ -557,12 +615,11 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         """Get research result without clearing it (for panel use)."""
         user = _require_user(request)
         _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
+        owned_disk_path = _require_owned_or_active_research_path(session_id, user)
-            raise HTTPException(404, "No research found for this session")
         result = research_handler.get_result(session_id)
         if result is None:
-            p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+            p = owned_disk_path
-            if p.exists():
+            if p is not None:
                 d = json.loads(p.read_text(encoding="utf-8"))
                 return {
                     "result": d.get("result", ""),
@@ -591,8 +648,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         # otherwise any authenticated user could spin off (and thereby read)
         # another user's report by guessing its session ID. Mirrors every other
         # endpoint in this file (see result_peek above).
-        if not _owns_in_memory(session_id, user):
+        owned_disk_path = _require_owned_or_active_research_path(session_id, user)
-            raise HTTPException(404, "No research found for this session")
         if session_manager is None:
             raise HTTPException(500, "session_manager not configured")
 
@@ -601,8 +657,8 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         sources = research_handler.get_sources(session_id) or []
         query = ""
 
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        path = owned_disk_path
-        if path.exists():
+        if path is not None:
             try:
                 disk = json.loads(path.read_text(encoding="utf-8"))
                 if not result:

routes/session_routes.py

@@ -13,6 +13,7 @@ from src.request_models import SessionResponse
 from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive
 from src.auth_helpers import effective_user, _auth_disabled, owner_filter
 from src.session_actions import is_session_recently_active
+from src.upload_handler import reserve_message_upload_references
 
 
 def _sanitize_export_filename(name: str) -> str:
@@ -203,10 +204,16 @@ def _pick_endpoint_for_sort(owner=None):
         return url, model, headers
     return None, None, None
 
-def setup_session_routes(session_manager: SessionManager, config: dict, webhook_manager=None):
+def setup_session_routes(
+    session_manager: SessionManager,
+    config: dict,
+    webhook_manager=None,
+    upload_handler=None,
+):
     """Setup session routes with the provided manager and config"""
 
     REQUEST_TIMEOUT = config.get("REQUEST_TIMEOUT", 20)
+    SESSION_MODEL_VALIDATION_TIMEOUT = min(float(REQUEST_TIMEOUT or 20), 3.0)
     OPENAI_API_KEY = config.get("OPENAI_API_KEY")
     SESSIONS_FILE = config.get("SESSIONS_FILE")
     
@@ -374,7 +381,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             from src.llm_core import list_model_ids
             ids = list_model_ids(
                 endpoint_url,
-                timeout=REQUEST_TIMEOUT,
+                timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
                 headers=validation_headers,
                 owner=user,
                 endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -394,7 +401,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             req_base = _os.path.basename(model_to_use.rstrip("/"))
             avail = list_model_ids(
                 endpoint_url,
-                timeout=REQUEST_TIMEOUT,
+                timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
                 headers=validation_headers,
                 owner=user,
                 endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -536,6 +543,22 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         body = await request.json()
         messages = body.get("messages", [])
         from core.models import ChatMessage
+        owner = effective_user(request)
+        try:
+            for message in messages:
+                missing_id = reserve_message_upload_references(
+                    upload_handler,
+                    owner,
+                    message.get("content"),
+                    message.get("metadata"),
+                )
+                if missing_id:
+                    raise HTTPException(
+                        409,
+                        f"Referenced upload is no longer available: {missing_id}",
+                    )
+        except (AttributeError, TypeError, ValueError) as exc:
+            raise HTTPException(400, "Invalid message attachment metadata") from exc
         for m in messages:
             sess.add_message(ChatMessage(m["role"], m["content"], metadata=m.get("metadata")))
         session_manager.save_sessions()

routes/shell_routes.py

@@ -164,6 +164,8 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
         return bool(binaries.get("llama-server") or dists.get("llama-cpp-python"))
     if name == "sglang":
         return bool(dists.get("sglang") or modules.get("sglang", {}).get("real_module"))
+    if name == "mlx_lm":
+        return bool(dists.get("mlx-lm") or modules.get("mlx_lm", {}).get("real_module"))
     if name == "diffusers":
         return bool(
             (dists.get("diffusers") or modules.get("diffusers", {}).get("real_module"))
@@ -210,6 +212,10 @@ def _package_status_note(name: str, probe: dict) -> str:
         if _package_installed_from_probe(name, probe):
             return f"diffusers {dists.get('diffusers', 'available')} with torch {dists.get('torch', 'available')}"
         return "Diffusers serving needs both diffusers and torch."
+    if name == "mlx_lm":
+        if _package_installed_from_probe(name, probe):
+            return f"MLX LM {dists.get('mlx-lm', 'available')}"
+        return "MLX serving needs mlx-lm on an Apple Silicon Mac."
     if name in dists:
         return f"{name} {dists[name]}"
     return ""
@@ -307,12 +313,14 @@ dist_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-cpp-python'],
     'sglang':['sglang'],
+    'mlx_lm':['mlx-lm'],
     'diffusers':['diffusers','torch'],
     'hf_transfer':['hf-transfer','hf_transfer'],
 }}
 bin_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-server'],
+    'tmux':['tmux'],
 }}
 
 def add_user_install_bins_to_path():
@@ -325,6 +333,8 @@ def add_user_install_bins_to_path():
     candidates.append(os.path.expanduser('~/llama.cpp/build/bin'))
     candidates.append(os.path.expanduser('~/llama.cpp/build-vulkan/bin'))
     candidates.append(os.path.expanduser('~/.local/bin'))
+    candidates.append('/opt/homebrew/bin')
+    candidates.append('/usr/local/bin')
     parts = os.environ.get('PATH', '').split(os.pathsep) if os.environ.get('PATH') else []
     changed = False
     for path in reversed([p for p in candidates if p]):
@@ -399,6 +409,47 @@ class ShellExecRequest(BaseModel):
     use_tmux: bool = False  # run in tmux session (survives browser disconnect)
 
 
+_REMOTE_TMUX_PATH_PREFIX = 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; '
+
+
+def _normalize_legacy_remote_tmux_exec(command: str) -> str:
+    """Repair stale frontend Cookbook tmux SSH commands.
+
+    Older loaded JS sends `ssh host 'tmux capture-pane ...'`. On macOS/Homebrew
+    remotes, non-login SSH shells often lack /opt/homebrew/bin, so tmux is
+    installed but the capture/kill command returns nothing. Keep this narrowly
+    scoped to SSH commands whose remote shell starts with `tmux `.
+    """
+    cmd = command or ""
+    if _REMOTE_TMUX_PATH_PREFIX in cmd or not cmd.lstrip().startswith("ssh "):
+        return cmd
+    try:
+        parts = shlex.split(cmd)
+    except Exception:
+        return cmd
+    if not parts or parts[0] != "ssh":
+        return cmd
+    remote_idx = -1
+    i = 1
+    while i < len(parts):
+        part = parts[i]
+        if part in {"-p", "-o", "-i", "-F", "-J", "-l", "-S", "-W", "-b", "-c", "-m"}:
+            i += 2
+            continue
+        if part.startswith("-"):
+            i += 1
+            continue
+        remote_idx = i
+        break
+    if remote_idx < 0 or remote_idx + 1 >= len(parts):
+        return cmd
+    remote_cmd = " ".join(parts[remote_idx + 1:]).strip()
+    if not remote_cmd.startswith("tmux "):
+        return cmd
+    repaired = parts[:remote_idx + 1] + [_REMOTE_TMUX_PATH_PREFIX + remote_cmd]
+    return shlex.join(repaired)
+
+
 async def _create_shell(command: str, **kwargs):
     """Spawn a shell subprocess for `command`.
 
@@ -815,6 +866,10 @@ def setup_shell_routes() -> APIRouter:
         if not cmd:
             return {"stdout": "", "stderr": "No command provided", "exit_code": 1}
 
+        fixed_cmd = _normalize_legacy_remote_tmux_exec(cmd)
+        if fixed_cmd != cmd:
+            logger.info("Rewrote legacy remote tmux exec command with Homebrew PATH")
+            cmd = fixed_cmd
         logger.info("User shell exec requested: length=%d", len(cmd))
         result = await _exec_shell(
             cmd, timeout=req.timeout if req.timeout is not None else EXEC_TIMEOUT
@@ -1134,6 +1189,13 @@ def setup_shell_routes() -> APIRouter:
                 "category": "LLM",
                 "target": "remote",
             },
+            {
+                "name": "mlx_lm",
+                "pip": "mlx-lm",
+                "desc": "Serve MLX-format models on Apple Silicon Macs",
+                "category": "LLM",
+                "target": "remote",
+            },
             {
                 "name": "APFEL",
                 "pip": "",
@@ -1279,9 +1341,9 @@ def setup_shell_routes() -> APIRouter:
                 for name in all_system_names:
                     qn = shlex.quote(name)
                     checks.append(
-                        f"if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
+                        f"PATH=\"$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH\"; if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
                     )
-                checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || true")
+                checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || { [ \"$(uname -s 2>/dev/null)\" = \"Darwin\" ] && echo ID=macos; } || true")
                 inner = " ; ".join(checks)
                 argv = _ssh_base_argv(host, ssh_port) + [inner]
                 proc = await asyncio.create_subprocess_exec(
@@ -1538,6 +1600,7 @@ def setup_shell_routes() -> APIRouter:
             "onnxruntime",
             "hdbscan",
             "vllm",
+            "mlx-lm",
         }
         if pip_name not in known:
             return {"ok": False, "error": f"Unknown package: {pip_name}"}
@@ -1584,6 +1647,19 @@ def setup_shell_routes() -> APIRouter:
                 elif n == "g++": out += ["gcc-c++"]
                 else: out.append(n)
             return out
+        def _apk(names):
+            out = []
+            for n in names:
+                if n == "build-essential": out.append("build-base")
+                else: out.append(n)
+            return out
+        def _zypper(names):
+            out = []
+            for n in names:
+                if n == "build-essential": out += ["gcc-c++", "make"]
+                elif n == "g++": out.append("gcc-c++")
+                else: out.append(n)
+            return out
         def _brew(names):
             return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
         # Build a single shell snippet that detects the package manager and
@@ -1592,6 +1668,8 @@ def setup_shell_routes() -> APIRouter:
         apt_pkgs = " ".join(shlex.quote(p) for p in _apt(pkgs))
         pac_pkgs = " ".join(shlex.quote(p) for p in _pacman(pkgs))
         dnf_pkgs = " ".join(shlex.quote(p) for p in _dnf(pkgs))
+        apk_pkgs = " ".join(shlex.quote(p) for p in _apk(pkgs))
+        zypper_pkgs = " ".join(shlex.quote(p) for p in _zypper(pkgs))
         brew_pkgs = " ".join(shlex.quote(p) for p in _brew(pkgs))
         # Error messages go to stderr (>&2) so the route's error field
         # gets populated. Without the redirect, `echo "ERROR…"` on stdout
@@ -1599,18 +1677,28 @@ def setup_shell_routes() -> APIRouter:
         # bare "HTTP 200" instead of surfacing the real reason.
         script = (
             'set -e; '
-            'if ! sudo -n true 2>/dev/null; then '
+            'BREW="$(command -v brew 2>/dev/null || true)"; '
-            '  echo "ERROR: passwordless sudo unavailable on this target. Run once: sudo apt install -y ' + " ".join(pkgs) + ' (or your distro equivalent: pacman -S, dnf install, brew install). After that, Cookbook can install the rest." >&2; exit 2; fi; '
+            'if [ -z "$BREW" ] && [ -x /opt/homebrew/bin/brew ]; then BREW=/opt/homebrew/bin/brew; fi; '
-            'if command -v apt-get >/dev/null 2>&1; then '
+            'if [ -z "$BREW" ] && [ -x /usr/local/bin/brew ]; then BREW=/usr/local/bin/brew; fi; '
-            f'  sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq && sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
+            'if [ -n "$BREW" ]; then '
-            'elif command -v pacman >/dev/null 2>&1; then '
+            f'  if [ -z "{brew_pkgs}" ]; then echo "Nothing to install with brew for requested packages." >&2; exit 4; fi; "$BREW" install {brew_pkgs}; exit $?; '
-            f'  sudo -n pacman -Sy --needed --noconfirm {pac_pkgs}; '
+            'fi; '
-            'elif command -v dnf >/dev/null 2>&1; then '
+            'if [ "$(id -u)" = "0" ]; then SUDO=""; '
-            f'  sudo -n dnf install -y {dnf_pkgs}; '
+            'elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then SUDO="sudo -n"; '
-            'elif command -v brew >/dev/null 2>&1; then '
-            f'  brew install {brew_pkgs}; '
             'else '
-            '  echo "ERROR: no supported package manager (apt/pacman/dnf/brew) on this target." >&2; exit 3; fi'
+            '  echo "ERROR: this target needs sudo for its OS package manager, but passwordless sudo is unavailable. Open a terminal on the target and run the shown install command once, then retry in Cookbook." >&2; exit 2; fi; '
+            'if command -v apt-get >/dev/null 2>&1; then '
+            f'  $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq && $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
+            'elif command -v pacman >/dev/null 2>&1; then '
+            f'  $SUDO pacman -Sy --needed --noconfirm {pac_pkgs}; '
+            'elif command -v dnf >/dev/null 2>&1; then '
+            f'  $SUDO dnf install -y {dnf_pkgs}; '
+            'elif command -v apk >/dev/null 2>&1; then '
+            f'  $SUDO apk add --no-interactive {apk_pkgs}; '
+            'elif command -v zypper >/dev/null 2>&1; then '
+            f'  $SUDO zypper --non-interactive install {zypper_pkgs}; '
+            'else '
+            '  echo "ERROR: no supported package manager (apt/pacman/dnf/apk/zypper/brew) on this target." >&2; exit 3; fi'
         )
         try:
             if host:

routes/task_routes.py

@@ -11,10 +11,14 @@ from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
 from core.database import SessionLocal, ScheduledTask, TaskRun
-from core.middleware import INTERNAL_TOOL_USER
 from core.constants import internal_api_base
 from src.auth_helpers import get_current_user
 from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
+from src.task_action_policy import (
+    ADMIN_ONLY_TASK_ACTIONS,
+    is_admin_only_task_action,
+    owner_has_admin_task_privileges,
+)
 from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS
 from routes.prefs_routes import _load_for_user, _save_for_user
 
@@ -417,28 +421,18 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 db.close()
         return {"ok": True, "opened": True, "enabled": bool(prefs.get("tasks_enabled")), "resumed": resumed}
 
-    # Actions that execute shell/SSH commands — restricted to admins.
+    # Actions that execute shell/SSH commands or cross into admin-only
+    # Cookbook serving surfaces — restricted to admins.
     # Non-admin users cannot create tasks with these action types via the
     # API. See review CRIT-C.
-    _ADMIN_ONLY_ACTIONS = {"run_local", "run_script", "ssh_command"}
+    _ADMIN_ONLY_ACTIONS = ADMIN_ONLY_TASK_ACTIONS
 
     def _is_admin(user: str | None) -> bool:
-        if not user:
+        return owner_has_admin_task_privileges(user)
-            return False
+
-        # In-process tool-loopback marker — AuthMiddleware validated
+    def _require_admin_for_task_action(user: str | None, task_type: str | None, action: str | None) -> None:
-        # the internal token + loopback client before stamping this,
+        if is_admin_only_task_action(task_type, action) and not _is_admin(user):
-        # so treat as admin-equivalent.
+            raise HTTPException(403, f"Action '{action}' requires admin privileges")
-        if user == INTERNAL_TOOL_USER:
-            return True
-        try:
-            from core.auth import AuthManager
-            auth = AuthManager()
-            if not auth.is_configured:
-                # Unconfigured single-user deploy: trust the local owner.
-                return True
-            return bool(auth.is_admin(user))
-        except Exception:
-            return False
 
     def _validate_then_task_id(db, then_task_id: Optional[str], user: Optional[str], current_task_id: Optional[str] = None) -> Optional[str]:
         target_id = (then_task_id or "").strip()
@@ -466,8 +460,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         # Block shell-executing action types for non-admins. action_run_local
         # uses subprocess.run(shell=True) and ssh_command / run_script run
         # arbitrary commands.
-        if req.task_type == "action" and req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
+        _require_admin_for_task_action(user, req.task_type, req.action)
-            raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
         if req.trigger_type == "schedule" and not req.schedule:
             raise HTTPException(400, "Schedule is required for schedule-triggered tasks")
         if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression:
@@ -681,6 +674,10 @@ def setup_task_routes(task_scheduler) -> APIRouter:
             if user and task.owner != user:
                 raise HTTPException(403, "Access denied")
 
+            next_task_type = req.task_type if req.task_type is not None else task.task_type
+            next_action = req.action if req.action is not None else task.action
+            _require_admin_for_task_action(user, next_task_type, next_action)
+
             if req.name is not None:
                 task.name = req.name
             if req.prompt is not None:
@@ -688,9 +685,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
             if req.task_type is not None:
                 task.task_type = req.task_type
             if req.action is not None:
-                # Same admin-only gate as create — see CRIT-C.
-                if req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
-                    raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
                 task.action = req.action
             if req.output_target is not None:
                 task.output_target = req.output_target
@@ -807,6 +801,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 raise HTTPException(404, "Task not found")
             if user and task.owner != user:
                 raise HTTPException(403, "Access denied")
+            _require_admin_for_task_action(user, task.task_type, task.action)
             task.status = "active"
             if (task.trigger_type or "schedule") == "schedule":
                 task.next_run = compute_next_run(
@@ -869,6 +864,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 raise HTTPException(404, "Task not found")
             if user and task.owner != user:
                 raise HTTPException(403, "Access denied")
+            _require_admin_for_task_action(user, task.task_type, task.action)
         finally:
             db.close()
         started = await task_scheduler.run_task_now(task_id, force=force)
@@ -1058,6 +1054,14 @@ def setup_task_routes(task_scheduler) -> APIRouter:
             ).first()
             if not task:
                 raise HTTPException(404, "Not found")
+            if (
+                is_admin_only_task_action(task.task_type, task.action)
+                and not owner_has_admin_task_privileges(task.owner)
+            ):
+                task.status = "paused"
+                task.next_run = None
+                db.commit()
+                raise HTTPException(403, f"Action '{task.action}' requires admin privileges")
         finally:
             db.close()
         started = await task_scheduler.run_task_now(task_id)

routes/upload_routes.py

@@ -10,16 +10,140 @@ from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
 from typing import List, Optional
 import logging
 from core.middleware import require_admin
-from core.database import SessionLocal, GalleryImage, Session as DbSession
+from core.database import (
+    SessionLocal,
+    ChatMessage as DbChatMessage,
+    CalendarCal,
+    CalendarEvent,
+    Document,
+    DocumentVersion,
+    GalleryImage,
+    Note,
+    Session as DbSession,
+)
 from src.auth_helpers import effective_user
+from src.attachment_refs import attachment_refs_from_metadata
 from src.constants import GENERATED_IMAGES_DIR
-from src.upload_handler import count_recent_uploads
+from src.upload_handler import (
+    UploadCleanupSafetyError,
+    count_recent_uploads,
+    extract_upload_ids,
+)
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/upload", tags=["upload"])
 UPLOAD_RESPONSE_HEADERS = {"X-Content-Type-Options": "nosniff"}
 
+def _upload_ids_from_persisted_text(value: object) -> set[str]:
+    """Return canonical upload IDs embedded in persisted text.
+
+    This covers attachment reference lines/URIs and the PDF source markers
+    stored by the document editor. False positives are intentionally
+    conservative: retaining an extra upload is safer than deleting referenced
+    bytes.
+    """
+    return extract_upload_ids(value)
+
+
+def _upload_ids_from_message_metadata(raw_metadata: object) -> set[str]:
+    """Extract attachment IDs from a persisted chat metadata JSON value.
+
+    Malformed metadata raises instead of being treated as an empty reference
+    set. The admin cleanup route catches that failure and aborts cleanup.
+    """
+    if raw_metadata in (None, ""):
+        return set()
+    if isinstance(raw_metadata, str):
+        metadata = json.loads(raw_metadata)
+    else:
+        metadata = raw_metadata
+    if not isinstance(metadata, dict):
+        raise ValueError("chat message metadata must be a JSON object")
+
+    attachments = metadata.get("attachments")
+    if attachments is not None:
+        if not isinstance(attachments, list) or any(
+            not isinstance(item, dict) for item in attachments
+        ):
+            raise ValueError("chat message attachments metadata is malformed")
+
+    ids = {
+        str(ref["attachment_id"])
+        for ref in attachment_refs_from_metadata(metadata)
+        if ref.get("attachment_id")
+    }
+    # Preserve canonical IDs even in older metadata shapes not normalized by
+    # attachment_refs_from_metadata().
+    ids.update(_upload_ids_from_persisted_text(json.dumps(metadata)))
+    return ids
+
+
+def _collect_persisted_upload_references() -> tuple[set[str], set[str]]:
+    """Collect upload IDs/hashes still referenced by durable application data.
+
+    The caller must treat any exception as an incomplete scan and fail closed.
+    There is no distinct artifact table in the current schema; artifact-like
+    attachment references persisted in chat/document text are covered by the
+    canonical-ID scan.
+    """
+    referenced_ids: set[str] = set()
+    referenced_hashes: set[str] = set()
+    db = SessionLocal()
+    try:
+        for content, raw_metadata in db.query(
+            DbChatMessage.content,
+            DbChatMessage.meta_data,
+        ).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+            referenced_ids.update(_upload_ids_from_message_metadata(raw_metadata))
+
+        for (content,) in db.query(Document.current_content).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+
+        for (content,) in db.query(DocumentVersion.content).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+
+        for filename, file_hash in db.query(
+            GalleryImage.filename,
+            GalleryImage.file_hash,
+        ).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(filename))
+            if file_hash:
+                referenced_hashes.add(str(file_hash))
+
+        for image_url, color, content, items in db.query(
+            Note.image_url,
+            Note.color,
+            Note.content,
+            Note.items,
+        ).yield_per(500):
+            for value in (image_url, color, content, items):
+                referenced_ids.update(_upload_ids_from_persisted_text(value))
+
+        for (color,) in db.query(CalendarCal.color).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(color))
+
+        for color, description, location in db.query(
+            CalendarEvent.color,
+            CalendarEvent.description,
+            CalendarEvent.location,
+        ).yield_per(500):
+            for value in (color, description, location):
+                referenced_ids.update(_upload_ids_from_persisted_text(value))
+
+        return referenced_ids, referenced_hashes
+    finally:
+        db.close()
+
+
+def _run_reference_safe_cleanup(upload_handler) -> int:
+    referenced_ids, referenced_hashes = _collect_persisted_upload_references()
+    return upload_handler.cleanup_old_uploads(
+        referenced_upload_ids=referenced_ids,
+        referenced_upload_hashes=referenced_hashes,
+    )
+
 def setup_upload_routes(upload_handler):
     """Setup upload routes with the provided handler"""
 
@@ -172,7 +296,9 @@ def setup_upload_routes(upload_handler):
                     "mime": meta["mime"],
                     "size": meta["size"],
                     "hash": meta["hash"],
+                    "checksum_sha256": meta.get("checksum_sha256") or meta["hash"],
                     "uploaded_at": meta["uploaded_at"],
+                    "created_at": meta.get("created_at") or meta["uploaded_at"],
                     "width": meta.get("width"),
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
@@ -195,7 +321,23 @@ def setup_upload_routes(upload_handler):
     async def manual_cleanup(request: Request):
         """Manually trigger cleanup of old uploads."""
         require_admin(request)
-        cleaned_count = upload_handler.cleanup_old_uploads()
+        try:
+            cleaned_count = await asyncio.to_thread(
+                _run_reference_safe_cleanup,
+                upload_handler,
+            )
+        except UploadCleanupSafetyError:
+            logger.exception("Upload cleanup aborted because index safety checks failed")
+            raise HTTPException(
+                503,
+                "Upload cleanup aborted because upload index integrity could not be verified",
+            )
+        except Exception:
+            logger.exception("Upload cleanup skipped because reference discovery failed")
+            raise HTTPException(
+                503,
+                "Upload cleanup skipped because persisted references could not be verified",
+            )
         return {"status": "success", "files_cleaned": cleaned_count}
 
     @router.get("/stats")

scripts/odysseus-backup

@@ -133,18 +133,32 @@ def cmd_list(args):
         emit([], args)
         return
     entries = []
-    for p in sorted(_BACKUP_DIR.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True):
+    for p in _BACKUP_DIR.iterdir():
-        if not p.is_file():
+        entry = _backup_entry(p)
-            continue
+        if entry is not None:
-        entries.append({
+            entries.append(entry)
-            "path": str(p),
+    entries.sort(key=lambda entry: entry["_mtime"], reverse=True)
-            "name": p.name,
+    for entry in entries:
-            "bytes": p.stat().st_size,
+        entry.pop("_mtime", None)
-            "modified": datetime.fromtimestamp(p.stat().st_mtime).isoformat(),
-        })
     emit(entries, args)
 
 
+def _backup_entry(p):
+    try:
+        if not p.is_file():
+            return None
+        st = p.stat()
+    except OSError:
+        return None
+    return {
+        "path": str(p),
+        "name": p.name,
+        "bytes": st.st_size,
+        "modified": datetime.fromtimestamp(st.st_mtime).isoformat(),
+        "_mtime": st.st_mtime,
+    }
+
+
 def cmd_verify(args):
     """Open the tarball read-only and walk its members — confirms
     integrity without extracting anything."""

scripts/odysseus-memory

@@ -90,7 +90,7 @@ def cmd_add(args):
     # add_entry doesn't save by default — the call in chat does it
     # after dedup checks. Persist here so a one-shot CLI add sticks.
     all_entries = _manager().load_all()
-    if not any(e.get("id") == entry.get("id") for e in all_entries):
+    if not any(isinstance(e, dict) and e.get("id") == entry.get("id") for e in all_entries):
         all_entries.append(entry)
         _manager().save(all_entries)
     emit(entry, args)

services/hwfit/fit.py

@@ -168,6 +168,19 @@ def _canonical_cpu_backend(system):
     return "cpu_x86"
 
 
+def _is_mlx_model(model, native_q=None):
+    name = (model.get("name") or "").lower()
+    provider = (model.get("provider") or "").lower()
+    fmt = (model.get("format") or "").lower()
+    q = (native_q if native_q is not None else _native_quant(model)).lower()
+    return (
+        q.startswith("mlx-")
+        or provider == "mlx-community"
+        or fmt == "mlx"
+        or name.startswith("mlx-community/")
+    )
+
+
 def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
     """Estimate tok/s. Uses active params for MoE (only active experts run per token).
 
@@ -313,6 +326,22 @@ def _fit_score(required, available):
     return 50
 
 
+def _is_unified_memory_system(system):
+    backend = (system.get("backend") or "").lower()
+    return bool(system.get("unified_memory")) or backend in ("metal", "mps", "apple")
+
+
+def _fit_level_for_budget(required_gb, budget_gb):
+    if not required_gb or not budget_gb or required_gb > budget_gb:
+        return "too_tight"
+    ratio = required_gb / budget_gb
+    if ratio <= 0.50:
+        return "perfect"
+    if ratio <= 0.78:
+        return "good"
+    return "marginal"
+
+
 def _context_score(ctx, use_case):
     target = CONTEXT_TARGET.get(use_case, 4096)
     if ctx >= target:
@@ -516,21 +545,42 @@ def analyze_model(model, system, target_quant=None, scoring_use_case=None, targe
     run_mode, quant, fit_ctx, required_gb = result
 
     # Determine fit level
-    budget = effective_vram if run_mode == "gpu" else available_ram
+    unified_memory = _is_unified_memory_system(system)
+    total_ram = system.get("total_ram_gb") or available_ram
+    unified_budget = max(total_ram or 0, available_ram or 0, effective_vram or 0)
+    budget = unified_budget if unified_memory else (effective_vram if run_mode == "gpu" else available_ram)
     if required_gb > budget:
         return None
     if run_mode == "gpu":
-        rec = model.get("recommended_ram_gb") or required_gb
+        if unified_memory:
-        if rec <= gpu_vram:
+            fit_level = _fit_level_for_budget(required_gb, budget)
-            fit_level = "perfect"
-        elif gpu_vram >= required_gb * 1.2:
-            fit_level = "good"
         else:
-            fit_level = "marginal"
+            # GPU-only fit must leave real allocator/KV/runtime headroom. The
+            # old check used recommended_ram_gb (or required_gb as a fallback),
+            # which made any model that barely fit VRAM read as "perfect".
+            # On CUDA/vLLM/SGLang that is misleading: 141 GB on a 160 GB box is
+            # runnable, but not a comfortable perfect fit.
+            if gpu_vram >= required_gb * 1.50:
+                fit_level = "perfect"
+            elif gpu_vram >= required_gb * 1.2:
+                fit_level = "good"
+            else:
+                fit_level = "marginal"
     elif run_mode == "cpu_offload":
-        fit_level = "good" if available_ram >= required_gb * 1.2 else "marginal"
+        fit_level = _fit_level_for_budget(required_gb, budget)
+        if fit_level == "perfect":
+            fit_level = "good"
     else:
-        fit_level = "marginal"
+        fit_level = _fit_level_for_budget(required_gb, budget)
+        if fit_level == "too_tight":
+            fit_level = "marginal"
+
+    # Rows that comfortably fit in a huge RAM/unified-memory pool should not all
+    # look "marginal"; that made 1B-70B CPU/Ollama rows orange on 256 GB systems.
+    if fit_level == "marginal" and budget and required_gb <= budget * 0.78:
+        fit_level = "good"
+    if fit_level == "good" and budget and required_gb <= budget * 0.50 and run_mode != "cpu_offload":
+        fit_level = "perfect"
 
     # Fraction of the model that spills to CPU RAM (drives the offload speed
     # model). When offloading, anything beyond the GPU's VRAM lives in system RAM.
@@ -621,6 +671,40 @@ SORT_KEYS = {
 }
 
 
+def _search_blob(*parts):
+    text = " ".join(str(p or "") for p in parts).lower()
+    compact = re.sub(r"[^a-z0-9]+", "", text)
+    spaced = re.sub(r"[^a-z0-9]+", " ", text).strip()
+    return f"{text} {spaced} {compact}"
+
+
+def _matches_search(model, search):
+    terms = [t for t in re.split(r"\s+", (search or "").strip().lower()) if t]
+    if not terms:
+        return True
+    blob = _search_blob(
+        model.get("name"),
+        model.get("provider"),
+        model.get("architecture"),
+        model.get("quantization"),
+        model.get("format"),
+        model.get("parameter_count"),
+    )
+    for term in terms:
+        norm = re.sub(r"[^a-z0-9]+", "", term)
+        if term not in blob and (not norm or norm not in blob):
+            if re.fullmatch(r"\d+(?:\.\d+)?b?", term):
+                try:
+                    wanted = float(term.rstrip("b"))
+                    actual = params_b(model)
+                except (TypeError, ValueError):
+                    actual = 0
+                if wanted > 0 and actual > 0 and abs(actual - wanted) <= max(5.0, wanted * 0.08):
+                    continue
+            return False
+    return True
+
+
 def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None, target_context=None, fit_only=False):
     """Rank all models against detected hardware. Returns sorted list of fit results.
 
@@ -693,10 +777,11 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
 
     for m in models:
         native_q = _native_quant(m)
+        is_mlx = _is_mlx_model(m, native_q)
 
-        # MLX needs the mlx_lm runtime, which Odysseus does not generate serve
+        # MLX is Apple Silicon-only. It should never appear on CUDA/ROCm/CPU,
-        # commands for. Hide it on every backend, including Metal.
+        # but it is first-class on Metal where mlx_lm.server can serve it.
-        if native_q.startswith("mlx-") or "mlx" in (m.get("name") or "").lower():
+        if is_mlx and not apple_silicon:
             continue
 
         # ROCm support for vLLM/SGLang quantized safetensors is too brittle to
@@ -723,7 +808,7 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
         # Windows is the same: Odysseus only supports llama.cpp on Windows,
         # which requires GGUF. vLLM/SGLang are explicitly blocked, so AWQ/GPTQ
         # models without a GGUF source are unservable there.
-        if (apple_silicon or consumer_amd or is_windows) and not (m.get("is_gguf") or m.get("gguf_sources")):
+        if (apple_silicon or consumer_amd or is_windows) and not is_mlx and not (m.get("is_gguf") or m.get("gguf_sources")):
             continue
 
         # Format filter: AWQ tab -> only AWQ models, FP4 tab -> FP4-family models, etc.
@@ -741,13 +826,26 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
             if quant in ("INT4", "INT8", "W4A16", "W8A8", "W8A16") and native_q != quant:
                 continue
 
-        if search:
+        if search and not _matches_search(m, search):
-            name = m.get("name", "").lower()
+            continue
-            provider = m.get("provider", "").lower()
-            if search.lower() not in name and search.lower() not in provider:
-                continue
 
-        result = analyze_model(m, system, target_quant=quant, scoring_use_case=(use_case or "general"), target_context=target_context)
+        model_quant = quant
+        # UI "Q4" means the user's looking for a 4-bit fit. On multi-GPU
+        # CUDA/vLLM/SGLang boxes, many practical 4-bit models are native AWQ
+        # safetensors, not GGUF Q4_K_M. If we pass Q4_K_M into a prequantized
+        # AWQ row, analyze_model correctly rejects it as the wrong serving
+        # format, but the result is confusing: highlighting Quant/Q4 hides the
+        # exact AWQ rows the machine is built to run. Treat Q4 as AWQ-4bit for
+        # native AWQ rows only on accelerator servers that can serve them.
+        if (
+            quant == "Q4_K_M"
+            and system.get("gpu_count", 1) >= 2
+            and not (apple_silicon or consumer_amd or is_windows)
+            and native_q == "AWQ-4bit"
+        ):
+            model_quant = native_q
+
+        result = analyze_model(m, system, target_quant=model_quant, scoring_use_case=(use_case or "general"), target_context=target_context)
         if result is None:
             continue
 

services/hwfit/hf_discovery.py

@@ -0,0 +1,374 @@
+import json
+import os
+import re
+import time
+import urllib.parse
+import urllib.request
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+
+from src.constants import DATA_DIR
+
+
+HF_COLLECTIONS_URL = "https://huggingface.co/api/collections"
+HW_FIT_CACHE_DIR = Path(DATA_DIR) / "hwfit"
+MLX_COMMUNITY_CACHE = HW_FIT_CACHE_DIR / "mlx_community_models.json"
+HF_COLLECTION_MODELS_CACHE = HW_FIT_CACHE_DIR / "hf_collection_models.json"
+HF_COLLECTION_TTL_SECONDS = 24 * 3600
+
+
+HF_COLLECTION_SOURCES = (
+    {
+        "key": "mlx_community",
+        "owner": "mlx-community",
+        "provider": "mlx-community",
+        "repo_prefix": "mlx-community/",
+        "mlx_only": True,
+    },
+    {
+        "key": "zai_org",
+        "owner": "zai-org",
+        "provider": "zai-org",
+    },
+    {
+        "key": "deepseek_ai",
+        "owner": "deepseek-ai",
+        "provider": "deepseek-ai",
+    },
+    {
+        "key": "minimax_ai",
+        "owner": "MiniMaxAI",
+        "provider": "MiniMaxAI",
+    },
+    {
+        "key": "qwen",
+        "owner": "Qwen",
+        "provider": "Qwen",
+    },
+    {
+        "key": "stepfun_ai",
+        "owner": "stepfun-ai",
+        "provider": "stepfun-ai",
+    },
+    {
+        "key": "google",
+        "owner": "google",
+        "provider": "google",
+    },
+    {
+        "key": "openai",
+        "owner": "openai",
+        "provider": "openai",
+    },
+    {
+        "key": "mistralai",
+        "owner": "mistralai",
+        "provider": "mistralai",
+    },
+    {
+        "key": "meta_llama",
+        "owner": "meta-llama",
+        "provider": "meta-llama",
+    },
+    {
+        "key": "nousresearch",
+        "owner": "NousResearch",
+        "provider": "NousResearch",
+    },
+    {
+        "key": "moonshotai",
+        "owner": "moonshotai",
+        "provider": "moonshotai",
+    },
+    {
+        "key": "mllama",
+        "owner": "mllama",
+        "provider": "mllama",
+    },
+)
+
+
+def _format_params(raw):
+    try:
+        n = int(raw or 0)
+    except (TypeError, ValueError):
+        n = 0
+    if n <= 0:
+        return "", 0
+    if n >= 1_000_000_000_000:
+        return f"{n / 1_000_000_000_000:.3g}T", n
+    if n >= 1_000_000_000:
+        return f"{n / 1_000_000_000:.4g}B", n
+    if n >= 1_000_000:
+        return f"{n / 1_000_000:.4g}M", n
+    if n >= 1_000:
+        return f"{n / 1_000:.4g}K", n
+    return str(n), n
+
+
+def _parse_params_from_name(repo_id):
+    name = (repo_id or "").rsplit("/", 1)[-1]
+    active = None
+    m_active = re.search(r"[-_][Aa](\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name)
+    if m_active:
+        active = int(float(m_active.group(1)) * 1_000_000_000)
+        name = name[: m_active.start()] + name[m_active.end() :]
+    total = None
+    for m in re.finditer(r"(\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name):
+        total = int(float(m.group(1)) * 1_000_000_000)
+        break
+    if total is None:
+        for m in re.finditer(r"(\d+(?:\.\d+)?)[Mm](?![a-zA-Z])", name):
+            total = int(float(m.group(1)) * 1_000_000)
+            break
+    return total or 0, active
+
+
+def _infer_quant(repo_id, source):
+    name = (repo_id or "").rsplit("/", 1)[-1].lower()
+    if source.get("mlx_only"):
+        if "8bit" in name or "8-bit" in name:
+            return "mlx-8bit"
+        if "6bit" in name or "6-bit" in name:
+            return "mlx-6bit"
+        if "5bit" in name or "5-bit" in name:
+            return "mlx-5bit"
+        if "3bit" in name or "3-bit" in name:
+            return "mlx-3bit"
+        if re.search(r"(^|[-_/])bf16($|[-_/])", name):
+            return "BF16"
+        return "mlx-4bit"
+    if "awq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
+        return "AWQ-8bit"
+    if "awq" in name or "4bit" in name or "4-bit" in name:
+        return "AWQ-4bit"
+    if "gptq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
+        return "GPTQ-Int8"
+    if "gptq" in name:
+        return "GPTQ-Int4"
+    if "mxfp4" in name or "nvfp4" in name or re.search(r"(^|[-_/])fp4($|[-_/])", name):
+        return "FP4-MoE-Mixed"
+    if "mxfp8" in name or re.search(r"(^|[-_/])fp8($|[-_/])", name):
+        return "FP8-Mixed"
+    if "gguf" in name or "q4_k" in name or "q4-k" in name:
+        return "Q4_K_M"
+    if re.search(r"(^|[-_/])bf16($|[-_/])", name):
+        return "BF16"
+    return "BF16"
+
+
+def _quant_bytes_per_param(quant):
+    return {
+        "BF16": 2.2,
+        "FP8": 1.15,
+        "FP8-Mixed": 1.15,
+        "FP4-MoE-Mixed": 0.62,
+        "AWQ-4bit": 0.62,
+        "AWQ-8bit": 1.15,
+        "GPTQ-Int4": 0.62,
+        "GPTQ-Int8": 1.15,
+        "Q4_K_M": 0.62,
+        "mlx-8bit": 1.25,
+        "mlx-6bit": 0.95,
+        "mlx-5bit": 0.82,
+        "mlx-4bit": 0.70,
+        "mlx-3bit": 0.55,
+    }.get(quant, 2.2)
+
+
+def _infer_context(repo_id, pipeline_tag):
+    text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
+    if any(k in text for k in ("whisper", "asr", "speech-recognition", "tts", "audio", "image", "video", "diffusion")):
+        return 4096
+    if any(k in text for k in ("glm-5.2", "deepseek-v4", "minimax-m3")):
+        return 1_000_000
+    if any(k in text for k in ("qwen3", "glm", "deepseek", "minimax")):
+        return 32768
+    return 32768
+
+
+def _infer_use_case(repo_id, pipeline_tag):
+    text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
+    if any(k in text for k in ("whisper", "asr", "speech-recognition", "transcrib")):
+        return "stt"
+    if any(k in text for k in ("tts", "text-to-speech", "kokoro", "audio")):
+        return "tts"
+    if any(k in text for k in ("image-text", "vision", "vlm", "vl-", "ocr", "multimodal")):
+        return "multimodal"
+    if any(k in text for k in ("code", "coder")):
+        return "coding"
+    if any(k in text for k in ("reason", "thinking", "thinker", "r1")):
+        return "reasoning"
+    return "general"
+
+
+def _entry_from_collection_item(collection, item, source):
+    repo_id = item.get("id") or ""
+    if item.get("type") != "model" or not repo_id:
+        return None
+    repo_prefix = source.get("repo_prefix")
+    if repo_prefix and not repo_id.startswith(repo_prefix):
+        return None
+    raw_params = item.get("numParameters") or 0
+    active = None
+    if not raw_params:
+        raw_params, active = _parse_params_from_name(repo_id)
+    param_label, raw_params = _format_params(raw_params)
+    if not raw_params:
+        return None
+
+    quant = _infer_quant(repo_id, source)
+    pipeline_tag = item.get("pipeline_tag") or ""
+    min_ram = round((raw_params / 1_000_000_000) * _quant_bytes_per_param(quant) + 0.8, 1)
+    last_modified = item.get("lastModified") or collection.get("lastUpdated") or ""
+    release_date = ""
+    if last_modified:
+        try:
+            release_date = parsedate_to_datetime(last_modified).date().isoformat()
+        except Exception:
+            release_date = str(last_modified)[:10]
+
+    entry = {
+        "name": repo_id,
+        "provider": source.get("provider") or repo_id.split("/", 1)[0],
+        "parameter_count": param_label,
+        "parameters_raw": raw_params,
+        "min_ram_gb": min_ram,
+        "recommended_ram_gb": round(min_ram * 1.3 + 0.5, 1),
+        "min_vram_gb": 0.0 if source.get("mlx_only") else min_ram,
+        "quantization": quant,
+        "context_length": _infer_context(repo_id, pipeline_tag),
+        "use_case": _infer_use_case(repo_id, pipeline_tag),
+        "capabilities": ["mlx"] if source.get("mlx_only") else ["vllm", "sglang"],
+        "pipeline_tag": pipeline_tag,
+        "architecture": "",
+        "hf_downloads": int(item.get("downloads") or 0),
+        "hf_likes": int(item.get("likes") or 0),
+        "release_date": release_date,
+        "format": "mlx" if source.get("mlx_only") else "safetensors",
+        "collection": collection.get("title") or "",
+        "description": collection.get("description") or "",
+        "_discovered": True,
+        "_source": "hf_collections",
+        "_source_owner": source.get("owner") or "",
+    }
+    if source.get("mlx_only"):
+        entry["mlx_only"] = True
+    if quant == "Q4_K_M":
+        entry["is_gguf"] = True
+        entry["format"] = "gguf"
+        entry["capabilities"] = ["llama.cpp"]
+    if active:
+        entry["is_moe"] = True
+        entry["active_parameters"] = active
+    return entry
+
+
+def _next_link(header):
+    if not header:
+        return None
+    m = re.search(r'<([^>]+)>;\s*rel="next"', header)
+    return m.group(1) if m else None
+
+
+def fetch_collection_models(source, timeout=20, max_pages=20):
+    params = urllib.parse.urlencode({
+        "owner": source["owner"],
+        "limit": "100",
+        "expand": "true",
+    })
+    url = f"{HF_COLLECTIONS_URL}?{params}"
+    models = {}
+    pages = 0
+    while url and pages < max_pages:
+        req = urllib.request.Request(url, headers={"User-Agent": "odysseus-hwfit/1.0"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            payload = json.load(resp)
+            url = _next_link(resp.headers.get("Link"))
+        pages += 1
+        if not isinstance(payload, list):
+            break
+        for collection in payload:
+            if not isinstance(collection, dict):
+                continue
+            for item in collection.get("items") or []:
+                if not isinstance(item, dict):
+                    continue
+                entry = _entry_from_collection_item(collection, item, source)
+                if entry and entry["name"] not in models:
+                    models[entry["name"]] = entry
+    rows = list(models.values())
+    rows.sort(key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""), reverse=True)
+    return rows
+
+
+def _load_cache(path):
+    try:
+        with path.open(encoding="utf-8") as f:
+            data = json.load(f)
+        rows = data.get("models") if isinstance(data, dict) else data
+        return rows if isinstance(rows, list) else []
+    except (OSError, ValueError):
+        return []
+
+
+def _write_cache(path, source, rows):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "source": source,
+        "fetched_at": int(time.time()),
+        "count": len(rows),
+        "models": rows,
+    }
+    tmp = path.with_suffix(".json.tmp")
+    tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def load_cached_mlx_community_models():
+    return _load_cache(MLX_COMMUNITY_CACHE)
+
+
+def load_cached_hf_collection_models():
+    return _load_cache(HF_COLLECTION_MODELS_CACHE)
+
+
+def _cache_fresh(path):
+    try:
+        return (time.time() - path.stat().st_mtime) < HF_COLLECTION_TTL_SECONDS
+    except OSError:
+        return False
+
+
+def refresh_mlx_community_cache(force=False):
+    if not force and _cache_fresh(MLX_COMMUNITY_CACHE):
+        return load_cached_mlx_community_models()
+    source = next(s for s in HF_COLLECTION_SOURCES if s["key"] == "mlx_community")
+    rows = fetch_collection_models(source)
+    _write_cache(MLX_COMMUNITY_CACHE, "https://huggingface.co/mlx-community/collections", rows)
+    return rows
+
+
+def refresh_hf_collection_models_cache(force=False):
+    if not force and _cache_fresh(HF_COLLECTION_MODELS_CACHE):
+        return load_cached_hf_collection_models()
+    rows_by_name = {}
+    for source in HF_COLLECTION_SOURCES:
+        if source["key"] == "mlx_community":
+            continue
+        try:
+            for row in fetch_collection_models(source):
+                rows_by_name.setdefault(row["name"], row)
+        except Exception:
+            # Keep partial refreshes useful. A temporary DNS/provider issue for
+            # one brand should not invalidate the other cached collection rows.
+            continue
+    rows = sorted(
+        rows_by_name.values(),
+        key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""),
+        reverse=True,
+    )
+    if rows:
+        _write_cache(HF_COLLECTION_MODELS_CACHE, "https://huggingface.co/collections", rows)
+        return rows
+    return load_cached_hf_collection_models()

services/hwfit/models.py

@@ -13,7 +13,7 @@ QUANT_BPP = {
     "AWQ-4bit": 0.50, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.50, "GPTQ-Int8": 1.0,
     "QAT-INT4": 0.50, "QAT-INT8": 1.0,
-    "mlx-4bit": 0.55, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
+    "mlx-3bit": 0.42, "mlx-4bit": 0.55, "mlx-5bit": 0.65, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
     # DeepSeek-V4-style mixed: MoE experts in FP4 (bulk), attention + non-
     # expert dense in FP8, embeddings/LM head in BF16. By weight count the
     # experts dominate so the effective BPP sits closer to FP4 than FP8.
@@ -32,7 +32,7 @@ QUANT_SPEED_MULT = {
     "AWQ-4bit": 1.2, "AWQ-8bit": 0.85,
     "GPTQ-Int4": 1.2, "GPTQ-Int8": 0.85,
     "QAT-INT4": 1.15, "QAT-INT8": 0.85,
-    "mlx-4bit": 1.15, "mlx-8bit": 0.85, "mlx-6bit": 1.0,
+    "mlx-3bit": 1.25, "mlx-4bit": 1.15, "mlx-5bit": 1.05, "mlx-6bit": 1.0, "mlx-8bit": 0.85,
     "FP4-MoE-Mixed": 1.10,  # slightly slower than pure FP4 because of mixed-dtype dispatch
     "FP8-Mixed": 0.85,
 }
@@ -53,7 +53,7 @@ QUANT_QUALITY_PENALTY = {
     # QAT-INT4 build lands far closer to bf16 than a post-training Q4/INT4
     # (Google reports near-bf16 quality). Penalize it lightly, not like Q4_K_M.
     "QAT-INT4": -1.0, "QAT-INT8": 0.0,
-    "mlx-4bit": -4.0, "mlx-8bit": -0.5, "mlx-6bit": -1.5,
+    "mlx-3bit": -8.0, "mlx-4bit": -4.0, "mlx-5bit": -2.5, "mlx-6bit": -1.5, "mlx-8bit": -0.5,
     # DeepSeek-V4 mixed: only MoE experts at FP4 (the rest is FP8/BF16),
     # so the realized quality is much closer to FP8 than to pure FP4 —
     # the activation-sensitive layers stay high-precision. ~0 penalty.
@@ -70,7 +70,7 @@ QUANT_BYTES_PER_PARAM = {
     "AWQ-4bit": 0.5, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.5, "GPTQ-Int8": 1.0,
     "QAT-INT4": 0.5, "QAT-INT8": 1.0,
-    "mlx-4bit": 0.5, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
+    "mlx-3bit": 0.375, "mlx-4bit": 0.5, "mlx-5bit": 0.625, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
     "FP4-MoE-Mixed": 0.55,
     "FP8-Mixed": 1.0,
 }
@@ -87,8 +87,11 @@ PREQUANTIZED_PREFIXES = (
 
 def infer_quantization_from_name(name):
     n = (name or "").lower()
+    model_name = n.rsplit("/", 1)[-1]
     if "nvfp4" in n:
         return "NVFP4"
+    if re.search(r"(^|[-_/])bf16($|[-_/])", model_name):
+        return "BF16"
     if "mxfp4" in n:
         return "MXFP4"
     if re.search(r"(^|[-_/])nf4($|[-_/])", n):
@@ -106,8 +109,12 @@ def infer_quantization_from_name(name):
         return "AWQ-8bit" if is8 else "AWQ-4bit"
     if "gptq" in n:
         return "GPTQ-Int8" if is8 else "GPTQ-Int4"
-    if "mlx" in n:
+    if n.startswith("mlx-community/") or "mlx" in model_name:
-        if "6bit" in n:
+        if "3bit" in model_name:
+            return "mlx-3bit"
+        if "5bit" in model_name:
+            return "mlx-5bit"
+        if "6bit" in model_name:
             return "mlx-6bit"
         return "mlx-8bit" if is8 else "mlx-4bit"
     if "fp8" in n:
@@ -138,7 +145,7 @@ def is_prequantized(model):
         or re.search(r"(^|[-_/])fp8($|[-_/\s])", text) is not None
         or (not (model.get("is_gguf") or model.get("gguf_sources")) and re.search(r"(^|[-_/])(?:int)?8bit($|[-_/\s])", text) is not None)
         or any(x in text for x in ("awq", "gptq", "mlx"))
-        or any(q.startswith(p) for p in PREQUANTIZED_PREFIXES)
+        or any(isinstance(q, str) and q.startswith(p) for p in PREQUANTIZED_PREFIXES)
     )
 
 
@@ -148,7 +155,7 @@ def params_b(model):
         return raw / 1_000_000_000.0
 
     pc = model.get("parameter_count", "")
-    if pc:
+    if isinstance(pc, str) and pc:
         pc = pc.strip().upper()
         m = re.match(r"^([\d.]+)\s*([BKMGT]?)$", pc)
         if m:
@@ -260,15 +267,75 @@ def infer_use_case(model):
 
 _models_cache = None
 
+def _load_model_file(path):
+    try:
+        with open(path, encoding="utf-8") as f:
+            loaded = json.load(f)
+            return loaded if isinstance(loaded, list) else []
+    except (FileNotFoundError, json.JSONDecodeError):
+        return []
+
+def reset_model_cache():
+    global _models_cache
+    _models_cache = None
+
+def refresh_dynamic_catalogs(force=False):
+    """Refresh API-backed model catalogs and invalidate the merged cache.
+
+    The bundled JSON files remain the offline fallback. Dynamic catalogs live
+    under DATA_DIR so runtime refreshes do not dirty the source tree.
+    """
+    from services.hwfit.hf_discovery import (
+        refresh_hf_collection_models_cache,
+        refresh_mlx_community_cache,
+    )
+
+    refreshed = {
+        "mlx_community": len(refresh_mlx_community_cache(force=force)),
+        "hf_collections": len(refresh_hf_collection_models_cache(force=force)),
+    }
+    reset_model_cache()
+    return refreshed
+
 def get_models():
     global _models_cache
     if _models_cache is None:
         data_path = os.path.join(os.path.dirname(__file__), "data", "hf_models.json")
+        static_mlx_path = os.path.join(os.path.dirname(__file__), "data", "mlx_community_models.json")
         try:
-            with open(data_path, encoding="utf-8") as f:
+            from services.hwfit.hf_discovery import (
-                _models_cache = [_normalize_model_entry(m) for m in json.load(f)]
+                load_cached_hf_collection_models,
-        except (FileNotFoundError, json.JSONDecodeError):
+                load_cached_mlx_community_models,
-            _models_cache = []
+            )
+            dynamic_mlx_models = load_cached_mlx_community_models()
+            dynamic_hf_models = load_cached_hf_collection_models()
+        except Exception:
+            dynamic_mlx_models = []
+            dynamic_hf_models = []
+        seen = set()
+        rows = []
+        def _append_models(models):
+            for model in models:
+                if not isinstance(model, dict):
+                    continue
+                name = model.get("name")
+                if not name or name in seen:
+                    continue
+                seen.add(name)
+                rows.append(_normalize_model_entry(model))
+
+        for model in _load_model_file(data_path):
+            if not isinstance(model, dict):
+                continue
+            name = model.get("name")
+            if not name or name in seen:
+                continue
+            seen.add(name)
+            rows.append(_normalize_model_entry(model))
+        _append_models(dynamic_hf_models)
+        _append_models(dynamic_mlx_models)
+        _append_models(_load_model_file(static_mlx_path))
+        _models_cache = rows
     return _models_cache
 
 

services/search/content.py

@@ -8,11 +8,13 @@ import os
 import re
 import logging
 import socket
+import ssl
 from datetime import datetime, timedelta
-from typing import List
+from typing import Iterable, List, cast
 from urllib.parse import urljoin, urlparse
 
 import httpx
+import httpcore
 from bs4 import BeautifulSoup
 
 from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
@@ -91,6 +93,148 @@ def _public_http_url(url: str) -> bool:
         return False
 
 
+def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname:
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    host = (parsed.hostname or "").strip().lower()
+    if host in ("localhost", "metadata", "metadata.google.internal"):
+        raise httpx.RequestError(f"Blocked non-public hostname: {host}")
+    try:
+        ip = ipaddress.ip_address(host)
+        if _is_private_address(ip):
+            raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
+        return [ip]
+    except httpx.RequestError:
+        raise
+    except ValueError:
+        pass
+    addrs = _resolve_hostname_ips(host)
+    if not addrs or any(_is_private_address(a) for a in addrs):
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    return addrs
+
+
+class _PinnedBackend(httpcore.NetworkBackend):
+    """Network backend that connects to a pre-resolved IP.
+
+    httpcore derives the TLS SNI and the ``Host`` header from the URL's
+    origin, not from the host argument passed to ``connect_tcp``. So
+    routing the TCP connect to a resolved IP while leaving the URL
+    untouched keeps SNI / vhost behaviour correct and closes the
+    DNS-rebinding TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress):
+        self._ip = str(ip)
+        self._real = httpcore.SyncBackend()
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options=None,
+    ):
+        return self._real.connect_tcp(
+            self._ip, port, timeout, local_address, socket_options
+        )
+
+    def connect_unix_socket(self, path, timeout=None, socket_options=None):
+        return self._real.connect_unix_socket(path, timeout, socket_options)
+
+    def sleep(self, seconds: float) -> None:
+        return self._real.sleep(seconds)
+
+
+# Map httpcore exception classes to their httpx equivalents. Built
+# once at import time from the public exception classes; avoids any
+# import of httpx's private transport machinery. httpcore's
+# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
+# close and retry on its own) — we never expect to see it surface to
+# a transport caller, so it has no httpx counterpart here.
+_HTTPCORE_TO_HTTPX_EXC = {
+    httpcore.ConnectError: httpx.ConnectError,
+    httpcore.ConnectTimeout: httpx.ConnectTimeout,
+    httpcore.LocalProtocolError: httpx.LocalProtocolError,
+    httpcore.NetworkError: httpx.NetworkError,
+    httpcore.PoolTimeout: httpx.PoolTimeout,
+    httpcore.ProtocolError: httpx.ProtocolError,
+    httpcore.ProxyError: httpx.ProxyError,
+    httpcore.ReadError: httpx.ReadError,
+    httpcore.ReadTimeout: httpx.ReadTimeout,
+    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
+    httpcore.TimeoutException: httpx.TimeoutException,
+    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
+    httpcore.WriteError: httpx.WriteError,
+    httpcore.WriteTimeout: httpx.WriteTimeout,
+}
+
+
+class _PinnedTransport(httpx.BaseTransport):
+    """Transport that pins every TCP connect to a pre-resolved IP.
+
+    Uses only the public ``httpcore`` and ``httpx`` APIs — no
+    subclassing of ``httpx.HTTPTransport``, no reads of private
+    ``httpcore.ConnectionPool`` attributes, no imports from
+    ``httpx private transport internals``. The URL is passed through unchanged so SNI
+    / vhost work as if httpx had been given the hostname directly;
+    only the TCP destination is pinned, closing the DNS-rebinding
+    TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
+        self._pool = httpcore.ConnectionPool(
+            ssl_context=ssl.create_default_context(),
+            http1=True,
+            http2=http2,
+            network_backend=_PinnedBackend(ip),
+        )
+
+    def __enter__(self):
+        self._pool.__enter__()
+        return self
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
+        self._pool.__exit__(exc_type, exc_value, traceback)
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        httpcore_req = httpcore.Request(
+            method=request.method,
+            url=httpcore.URL(
+                scheme=request.url.raw_scheme,
+                host=request.url.raw_host,
+                port=request.url.port,
+                target=request.url.raw_path,
+            ),
+            headers=request.headers.raw,
+            content=request.stream,
+            extensions=request.extensions,
+        )
+        try:
+            httpcore_resp = self._pool.handle_request(httpcore_req)
+            # Eager materialisation matches the original
+            # ``response.text`` usage in fetch_webpage_content. The
+            # sync pool's stream is a plain Iterable[bytes] despite
+            # the httpcore type hint unioning the async variant.
+            content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
+        except Exception as exc:
+            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
+            if mapped is not None:
+                raise mapped(str(exc)) from exc
+            raise
+
+        return httpx.Response(
+            status_code=httpcore_resp.status,
+            headers=httpcore_resp.headers,
+            content=content,
+            extensions=httpcore_resp.extensions,
+        )
+
+    def close(self) -> None:
+        self._pool.close()
+
 class BodyTooLargeError(Exception):
     """The server declared a body larger than the hard fetch ceiling."""
 
@@ -141,78 +285,78 @@ class _CappedFetch:
 
 def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
                     max_bytes: int = None) -> "_CappedFetch":
-    """Capped streaming GET with SSRF-guarded manual redirects.
+    """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
 
-    The body is streamed and buffering stops at ``max_bytes`` (default: the
+    Each hop is resolved once, validated as public, and then the actual TCP
-    soft cap), so an oversized resource cannot be pulled into memory or the
+    connection is pinned to that resolved IP. The request URL is left unchanged
-    content cache in full. When Content-Length already declares a body over
+    so Host and TLS SNI keep the original hostname.
-    the hard ceiling, the fetch is refused before any body bytes are read.
     """
     cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
     current = url
     for _ in range(max_redirects + 1):
-        if not _public_http_url(current):
+        ips = _resolve_public_ips(current)
-            raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
+
         # Force identity transfer-encoding. With gzip/deflate the wire bytes
-        # (and Content-Length) can be a small fraction of the decoded body, so
+        # and Content-Length can be a small fraction of the decoded body, so a
-        # a tiny compressed response could pass the hard-cap preflight and then
+        # tiny compressed response could pass the hard-cap preflight and then
-        # expand past the ceiling in a single decoded chunk before the streamed
+        # expand past the ceiling in one decoded chunk before the streamed cap
-        # cap below can slice it. Identity makes Content-Length the true body
+        # below can slice it.
-        # size and keeps each streamed chunk bounded by the network read.
         req_headers = dict(headers or {})
         req_headers["Accept-Encoding"] = "identity"
-        with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
-                          follow_redirects=False) as response:
-            if response.status_code in (301, 302, 303, 307, 308):
-                location = response.headers.get("location")
-                if not location:
-                    return _CappedFetch(response.status_code, response.headers, b"",
-                                        False, None, response.encoding, str(response.url))
-                current = urljoin(str(response.url), location)
-                continue
 
-            # A server can ignore the identity request and still return a
+        with httpx.Client(
-            # compressed body; httpx.iter_bytes would then decode it, and a tiny
+            headers=req_headers,
-            # gzip can balloon into one decoded chunk far past the cap before we
+            timeout=timeout,
-            # slice. Refuse a compressed Content-Encoding so the streamed cap
+            follow_redirects=False,
-            # stays a real memory bound (Content-Length is the compressed wire
+            transport=_PinnedTransport(ips[0]),
-            # length here, so the preflight and size metadata are unreliable too).
+        ) as client:
-            enc = (response.headers.get("content-encoding") or "").strip().lower()
+            with client.stream("GET", current) as response:
-            if enc and enc != "identity":
+                if response.status_code in (301, 302, 303, 307, 308):
-                raise httpx.RequestError(
+                    location = response.headers.get("location")
-                    f"Refusing compressed response (Content-Encoding: {enc}) after "
+                    if not location:
-                    "requesting identity: cannot bound decoded body size",
+                        return _CappedFetch(response.status_code, response.headers, b"",
-                    request=httpx.Request("GET", current),
+                                            False, None, response.encoding, str(response.url))
-                )
+                    current = urljoin(str(response.url), location)
+                    continue
 
-            declared = None
+                # A server can ignore the identity request and still return a
-            raw_len = response.headers.get("content-length")
+                # compressed body; httpx.iter_bytes would then decode it, and a
-            if raw_len and raw_len.isdigit():
+                # tiny gzip can balloon into one decoded chunk far past the cap.
-                declared = int(raw_len)
+                # Refuse compressed Content-Encoding so the streamed cap stays
-            # Refuse before buffering anything when the server already tells
+                # a real memory bound.
-            # us the body exceeds the absolute ceiling (Content-Length is wire
+                enc = (response.headers.get("content-encoding") or "").strip().lower()
-            # bytes; the decompressed body can only be larger).
+                if enc and enc != "identity":
-            if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
+                    raise httpx.RequestError(
-                raise BodyTooLargeError(current, declared)
+                        f"Refusing compressed response (Content-Encoding: {enc}) after "
+                        "requesting identity: cannot bound decoded body size",
+                        request=httpx.Request("GET", current),
+                    )
+
+                declared = None
+                raw_len = response.headers.get("content-length")
+                if raw_len and raw_len.isdigit():
+                    declared = int(raw_len)
+
+                if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
+                    raise BodyTooLargeError(current, declared)
+
+                chunks = []
+                read = 0
+                truncated = False
+                for chunk in response.iter_bytes():
+                    read += len(chunk)
+                    if read > cap:
+                        keep = cap - (read - len(chunk))
+                        if keep > 0:
+                            chunks.append(chunk[:keep])
+                        truncated = True
+                        break
+                    chunks.append(chunk)
+
+                return _CappedFetch(response.status_code, response.headers,
+                                    b"".join(chunks), truncated, declared,
+                                    response.encoding, str(response.url))
 
-            chunks = []
-            read = 0
-            truncated = False
-            # We requested identity above, so iter_bytes yields the raw body in
-            # network-read-sized chunks (no decompression expansion); the cap
-            # therefore bounds what we actually buffer.
-            for chunk in response.iter_bytes():
-                read += len(chunk)
-                if read > cap:
-                    keep = cap - (read - len(chunk))
-                    if keep > 0:
-                        chunks.append(chunk[:keep])
-                    truncated = True
-                    break
-                chunks.append(chunk)
-            return _CappedFetch(response.status_code, response.headers,
-                                b"".join(chunks), truncated, declared,
-                                response.encoding, str(response.url))
     raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
 
 # PDF extraction (optional dependency)

services/search/query.py

@@ -34,8 +34,14 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
     cleaned = query
     if qtype:
         cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
-    for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
+    # Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
-        entities["names"].append(token)
+    # class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
+    # "São" (shredded). Keep the ASCII behaviour — the word boundary already
+    # excludes camelCase mid-word capitals — by requiring an all-alphabetic
+    # token of length > 1 whose first character is uppercase.
+    for token in re.findall(r"\b\w+\b", cleaned):
+        if len(token) > 1 and token[0].isupper() and token.isalpha():
+            entities["names"].append(token)
     for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
         entities["dates"].append(year)
     month_day_year = re.findall(

services/tts/tts_service.py

@@ -68,7 +68,7 @@ class TTSService:
         if provider == "local":
             kokoro = self._get_kokoro()
             return kokoro is not None and kokoro.available
-        if provider.startswith("endpoint:"):
+        if isinstance(provider, str) and provider.startswith("endpoint:"):
             return True  # assume reachable; errors surface at synthesis time
         return False
 

src/action_intents.py

@@ -92,8 +92,21 @@ _ROUTING_PATTERNS: tuple[tuple[str, str, Pattern[str]], ...] = tuple(
 
         # Deep research jobs, not quick conceptual mentions of research.
         ("web", "explicit web search request", rf"{_PLEASE}(?:do|run|use|perform|make)\s+(?:a\s+)?(?:web\s+search|search\s+the\s+web)\b.+"),
-        ("web", "web lookup imperative request", rf"{_PLEASE}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"),
+        ("web", "generic search request", rf"{_PLEASE}search\s+(?!(?:my\s+)?(?:chats?|history|sessions?|notes?|todos?|emails?|mail|inbox|documents?|docs|gallery|images?|files?)\b).+"),
-        ("web", "assistant web lookup request", rf"{_ACTION_QUESTION}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"),
+        ("web", "web lookup imperative request", rf"{_PLEASE}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google(?:\s+it)?)\b.*"),
+        ("web", "short web lookup follow-up", rf"{_PLEASE}(?:just\s+)?(?:look\s+it\s+up|look\s+up|search\s+(?:online|web|now)|search\s+it)\b\s*$"),
+        ("web", "assistant short web lookup request", rf"{_ACTION_QUESTION}(?:search|look\s+up|google)(?:\s+(?:online|web|now|it))?\b.*"),
+        ("web", "assistant web lookup request", rf"{_ACTION_QUESTION}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google(?:\s+it)?)\b.*"),
+        ("web", "assistant weather check request", rf"{_ACTION_QUESTION}(?:check|find|get|look\s+up)\b.{{0,100}}\b(?:weather|forecast)\b.*"),
+        ("web", "news lookup request", r"\b(?:news|headlines)\s+(?:in|from|about|for)\s+[\w\s.-]{2,80}\??\s*$"),
+        ("web", "forecast lookup request", r"\b(?:hourly|daily|weekly|local)\s+(?:weather\s+)?forecast\b|\b(?:weather\s+)?forecast\s+(?:for|today|tomorrow|now|hourly)\b"),
+        ("web", "weather lookup request", r"\bweather\b.{0,80}\b(?:hourly|rain|raining|rin|today|tomorrow|update|current|now)\b|\b(?:hourly|rain|raining|rin)\b.{0,80}\bweather\b"),
+        ("web", "rain lookup request", r"\b(?:hourly|daily|weekly|local|today|tomorrow|current|now|update)\b.{0,100}\b(?:rain|raining|rainy|precipitation|showers?)\b|\b(?:rain|raining|rainy|precipitation|showers?)\b.{0,100}\b(?:hourly|daily|weekly|local|today|tomorrow|current|now|update|in|for|at)\b"),
+        ("web", "bare weather lookup request", r"\b(?:weather|forecast)\s+(?:in|for|at)?\s*[\w\s.-]{2,80}\??\s*$|\b[\w\s.-]{2,80}\s+(?:weather|forecast)\??\s*$"),
+        ("web", "latest info lookup request", r"\b(?:latest|current|newest|recent|up(?: |-)?to(?: |-)?date)\s+(?:info|information|updates?|details?|developments?)\s+(?:on|about|for|in)\s+[\w\s.,:'\"/-]{2,120}\??\s*$"),
+        ("web", "current/latest lookup request", r"\b(?:current|latest|today'?s?|right\s+now|live|online)\b.{0,120}\b(?:rate|price|news|weather|forecast|score|exchange|market|status)\b"),
+        ("web", "rate/price/news lookup request", r"\b(?:rate|rates|price|prices|news|weather|forecast|score|exchange|currency|market)\b.{0,120}\b(?:now|today|current|latest|online|live|search|look\s+up|find)\b"),
+        ("web", "conversion-rate lookup request", r"\b(?:convert|conversion|exchange)\b.{0,120}\b(?:rate|rates|currency|currencies|price|prices)\b"),
         ("research", "deep research imperative request", rf"{_PLEASE}(?:research|deep\s+dive|look\s+into|investigate)\s+.+"),
         ("research", "assistant deep research request", rf"{_ACTION_QUESTION}(?:research|do\s+research|deep\s+dive|look\s+into|investigate)\s+.+"),
 

src/agent_loop.py

@@ -24,7 +24,7 @@ from src.model_context import estimate_tokens
 from src.settings import get_setting
 from src.prompt_security import untrusted_context_message
 from src.tool_security import blocked_tools_for_owner, plan_mode_disabled_tools
-from src.tool_policy import GUIDE_ONLY_DIRECTIVE, ToolPolicy
+from src.tool_policy import GUIDE_ONLY_DIRECTIVE, WEB_TOOL_NAMES, ToolPolicy
 from src.tool_utils import _truncate, get_mcp_manager
 from src.agent_tools import (
     parse_tool_blocks,
@@ -43,6 +43,44 @@ from src.agent_tools import (
 logger = logging.getLogger(__name__)
 
 
+def _looks_like_notes_list_request(text: str) -> bool:
+    """Whether the user is asking to see existing notes, not create one."""
+    t = (text or "").lower()
+    return bool(
+        re.search(r"\b(what|show|list|see|current|existing|all|my)\b.{0,60}\bnotes?\b", t)
+        or re.search(r"\bnotes?\b.{0,60}\b(what|show|list|see|current|existing|all|my)\b", t)
+    )
+
+
+def _note_list_summary_from_tool_output(raw: str, max_items: int = 20) -> str:
+    """Format manage_notes list/search output for chat without an LLM pass."""
+    if not isinstance(raw, str) or not raw.strip():
+        return ""
+    titles: list[str] = []
+    for line in raw.splitlines():
+        m = re.match(r"^\s*-\s+\[[^\]]+\]\s+\*\*(.*?)\*\*(.*)$", line)
+        if not m:
+            continue
+        title = re.sub(r"\s+", " ", m.group(1)).strip()
+        suffix = re.sub(r"\s+", " ", m.group(2) or "").strip()
+        label = f"{title} {suffix}".strip()
+        if label:
+            titles.append(label)
+        if len(titles) >= max_items:
+            break
+    if not titles:
+        if re.search(r"\b(no notes|0 notes|found 0)\b", raw, re.IGNORECASE):
+            return "No notes found."
+        return ""
+    total = len(re.findall(r"^\s*-\s+\[[^\]]+\]\s+\*\*", raw, re.MULTILINE))
+    heading_count = total or len(titles)
+    lines = [f"Here are your notes ({heading_count}):"]
+    lines.extend(f"- {title}" for title in titles)
+    if total and total > len(titles):
+        lines.append(f"- ...and {total - len(titles)} more")
+    return "\n".join(lines)
+
+
 def _load_mcp_disabled_map() -> Dict[str, set]:
     """Load per-server disabled tool sets from the database."""
     from core.database import McpServer, SessionLocal
@@ -73,9 +111,11 @@ _AGENT_RULES = """\
 ## Rules
 - Only use tools when needed. Don't search for things you already know.
 - For web lookup/search/latest/current requests, use `web_search` or `web_fetch`. Do NOT use `bash`, `python`, `curl`, `requests`, or scraping code for web lookup unless web tools are disabled or already failed.
+- If `web_search` is listed in this prompt, web search is available. Do NOT tell the user search/web tools are unavailable.
 - These exact tags execute automatically. For showing code examples, use ```shell, ```sh, ```py, etc. instead.
 - Multiple tool blocks per response OK. 60s timeout per tool, 10K char output limit.
 - Code/content >15 lines → ```create_document (NOT in chat). Short snippets OK in chat.
+- Long-form or structured writing is a document by default when the user asks to write/create/make/generate it and the answer would be more than a short paragraph. Use create_document instead of dumping the full content in chat.
 - Editing an existing document: ALWAYS use ```edit_document with FIND/REPLACE blocks. Do NOT rewrite the whole document with ```update_document unless genuinely changing more than half of it.
 - BIAS TOWARD ACTION on edit requests. If the user says "edit out X", "remove the Y paragraph", "change Z" — JUST DO IT with your best interpretation. Don't ask for clarification on minor ambiguity. The user can undo or re-prompt if wrong.
 - AFTER A TOOL SUCCEEDS, do not second-guess. The success message ("Document edited: v2, 1 edit") means it worked. Reply in ONE short sentence confirming what was done. No re-checking, no replaying the diff in your head, no validation theater.
@@ -120,8 +160,10 @@ _API_AGENT_RULES = """\
 - Only call tools when they materially help answer the request.
 - You MUST use tools to take action — do not describe what you would do. Act, don't narrate.
 - For web lookup/search/latest/current requests, call `web_search` or `web_fetch`. Do NOT use shell, Python, curl, requests, or scraping code for web lookup unless web tools are unavailable or already failed.
+- If `web_search` is listed in this prompt, web search is available. Do NOT tell the user search/web tools are unavailable.
 - Keep answers concise unless the user asks for depth.
 - For long code or content, use document tools instead of pasting large blocks into chat.
+- Long-form or structured writing is a document by default when the user asks to write/create/make/generate it and the answer would be more than a short paragraph. Call create_document instead of dumping the full content in chat.
 - Editing an existing document: ALWAYS use `edit_document` with find/replace. Only use `update_document` for genuine full rewrites (>50% changed) — do NOT echo the entire file back for small edits.
 - If the active editor document is an email draft/compose window, treat that open email as the target for "write this", "write the email", "reply with...", "make it say...", "draft this", and similar requests. Do NOT create another document, search/list/manage documents, or open a different reply unless the user explicitly asks. Edit the open email draft with `edit_document` or `update_document`; preserve To/Cc/Bcc/Subject/In-Reply-To/References/X-* header lines unless the user asks to change them.
 - "Give suggestions / feedback / review / how can I improve this / what would make it better" about the OPEN document → call `suggest_document`, do NOT write a prose list of ideas in chat. It creates inline accept/reject bubbles on the doc. Give concrete `find`/`replace`/`reason` items. To suggest an ADDITION (e.g. "add a bow to the SVG", a new section), set `find` to a short existing anchor snippet and `replace` to that same snippet PLUS the new content. Only answer in prose when no document is open, or the request is purely conceptual with no concrete change to propose.
@@ -279,7 +321,7 @@ _DOMAIN_RULES = {
 }
 
 _DOMAIN_TOOL_MAP = {
-    "web": {"web_search", "web_fetch", "trigger_research", "manage_research"},
+    "web": set(WEB_TOOL_NAMES),
     "documents": {"create_document", "edit_document", "update_document", "suggest_document", "manage_documents"},
     "email": {"list_email_accounts", "list_emails", "read_email", "send_email", "reply_to_email", "bulk_email", "archive_email", "delete_email", "mark_email_read", "resolve_contact", "manage_contact"},
     "cookbook": {"download_model", "serve_model", "serve_preset", "list_serve_presets", "list_served_models", "stop_served_model", "tail_serve_output", "list_downloads", "cancel_download", "search_hf_models", "list_cached_models", "list_cookbook_servers", "adopt_served_model"},
@@ -337,6 +379,7 @@ Or with JSON for fresh news:
 {"query": "<your query>", "time_filter": "day"}
 ```
 Search the web for a SINGLE quick fact/lookup mid-task. For news / "today" / "latest" queries, pass `time_filter` ("day", "week", "month", or "year"). NOT for "research X" / "do research on X" / "look into X" requests — those mean a multi-source DEEP RESEARCH job: use `trigger_research` instead (it runs in the Deep Research sidebar and produces a full report). web_search = one quick query; trigger_research = a researched report.
+If this `web_search` tool section is visible, search is available. Do NOT tell the user web/search tools are unavailable.
 Use this instead of `bash`, `curl`, `python`, `requests`, or scraping code for web lookup/search/latest/current requests.""",
 
     "web_fetch": """\
@@ -469,7 +512,7 @@ Bulk delete/archive/mark emails. Use this for "delete all those" after listing e
 {"action": "create_event", "summary": "<event title>", "dtstart": "<natural language or ISO datetime>"}
 ```
 Calendar event management (CalDAV). Actions: `list_events`, `create_event`, `update_event`, `delete_event`, `list_calendars`. \
-For `list_events`: {start?, end?, calendar?}; prefer `start`/`end` for the range, though start_date/end_date and from/to aliases are accepted. \
+For `list_events`: {action: "list_events", start: "YYYY-MM-DDT00:00:00", end: "YYYY-MM-DDT00:00:00", calendar?}; resolve month/week phrases yourself from the Current date and time context and do not pass a loose `query` field. Prefer `start`/`end`; start_time/end_time, start_date/end_date, and from/to aliases are accepted. \
 For `create_event`: {summary, dtstart, dtend?, duration?, calendar?, location?, description?, reminder_minutes?, rrule?}. \
 For `update_event`: {uid, summary?, dtstart?, dtend?, all_day?, location?, description?, event_type?, importance?, rrule?}. Pass `rrule: ""` to remove recurrence and make a repeating event a single event. \
 `dtstart` accepts natural language ("tomorrow at 1pm", "in 2 hours", "next monday 9am") or ISO ("2026-05-12T13:00:00"). \
@@ -614,16 +657,6 @@ def _assemble_prompt(tool_names: set, disabled_tools: set = None, compact: bool
     if one_liners:
         parts.append("## Additional tools\n" + "\n".join(one_liners))
 
-    # Mention tools that exist but weren't included
-    all_known = set(TOOL_SECTIONS.keys())
-    not_shown = all_known - included - disabled
-    if not_shown:
-        sample = sorted(not_shown)[:5]
-        hint = ", ".join(sample)
-        if len(not_shown) > 5:
-            hint += f", ... ({len(not_shown) - 5} more)"
-        parts.append(f"(Other tools available when needed: {hint})")
-
     parts.append(_AGENT_RULES)
     parts.extend(_domain_rules_for_tools(included))
     return "\n\n".join(parts)
@@ -763,6 +796,15 @@ def _extract_last_user_message(messages: List[Dict]) -> str:
     return ""
 
 
+def _user_turn_count(messages: List[Dict]) -> int:
+    """Count real user turns in the message list."""
+    count = 0
+    for msg in messages or []:
+        if msg.get("role") == "user":
+            count += 1
+    return count
+
+
 def _insert_before_latest_user(messages: List[Dict], context_msg: Dict) -> List[Dict]:
     """Insert a context message immediately before the latest user turn."""
     out = list(messages or [])
@@ -973,7 +1015,7 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o
         domains.add("cookbook")
     if has(r"\b(emails?|mails?|gmail|inbox|reply|forward|cc|bcc|send email|compose email|draft email|message chris|message him|message her)\b"):
         domains.add("email")
-    if has(r"\b(note|todo|to-do|checklist|task list|remind me|reminder|buy|pickup|pick up)\b"):
+    if has(r"\b(notes?|todos?|to-dos?|checklists?|task list|remind me|reminders?|buy|pickup|pick up)\b"):
         domains.add("notes_calendar_tasks")
     if has(r"\b(every day|every morning|every evening|recurring|automatically|cron|scheduled task|background task)\b"):
         domains.add("notes_calendar_tasks")
@@ -1076,6 +1118,20 @@ def _turn_targets_active_document(intent: Dict[str, object], last_user: str, act
         text,
     ):
         return True
+    if re.search(
+        r"\b(?:make|change|update|fix|edit|rewrite|rework|revise|replace|remove|delete|add|append|insert|set|turn)\b"
+        r".{0,80}\b(?:day\s*\d+|row|rows|column|columns|table|section|chapter|part|paragraph|line|lines|"
+        r"title|heading|body|intro|introduction|conclusion|schedule|itinerary|draft|content)\b",
+        text,
+    ):
+        return True
+    if re.search(
+        r"\b(?:day\s*\d+|row|rows|column|columns|table|section|chapter|part|paragraph|line|lines|"
+        r"title|heading|body|intro|introduction|conclusion|schedule|itinerary)\b"
+        r".{0,80}\b(?:make|change|update|fix|edit|rewrite|rework|revise|replace|remove|delete|add|append|insert|set|turn)\b",
+        text,
+    ):
+        return True
     if re.search(
         r"\b(?:add|insert|include|apply|put)\b.+\b(?:to it|to this|there|in it|in this|in the text|in the document)\b",
         text,
@@ -1098,6 +1154,18 @@ def _turn_targets_active_document(intent: Dict[str, object], last_user: str, act
     ))
 
 
+def _is_email_document_obj(active_document) -> bool:
+    if active_document is None:
+        return False
+    raw_doc = getattr(active_document, "current_content", "") or ""
+    title_l = (getattr(active_document, "title", "") or "").strip().lower()
+    return (
+        getattr(active_document, "language", None) == "email"
+        or title_l in {"new email", "new mail", "new message"}
+        or ("To:" in raw_doc[:400] and "Subject:" in raw_doc[:400] and "\n---\n" in raw_doc)
+    )
+
+
 def _minimal_saved_memory_message(messages: List[Dict]) -> Optional[Dict]:
     facts: List[str] = []
     seen = set()
@@ -1125,9 +1193,9 @@ def _minimal_saved_memory_message(messages: List[Dict]) -> Optional[Dict]:
                 continue
             seen.add(fact)
             facts.append(fact)
-            if len(facts) >= 12:
+            if len(facts) >= 8:
                 break
-        if len(facts) >= 12:
+        if len(facts) >= 8:
             break
     if not facts:
         return None
@@ -1144,6 +1212,41 @@ def _minimal_saved_memory_message(messages: List[Dict]) -> Optional[Dict]:
     }
 
 
+def _compact_email_draft_context(raw: str, *, max_own_chars: int = 1200, max_history_chars: int = 1200) -> str:
+    """Compact an email compose document for prompt injection.
+
+    The editor/backend preserve quoted history mechanically, so the model only
+    needs enough of the previous message to understand what to answer.
+    """
+    text = raw or ""
+    if "\n---\n" not in text:
+        return text[:3500] + ("\n...[truncated]" if len(text) > 3500 else "")
+    header, body = text.split("\n---\n", 1)
+    literal = "---------- Previous message ----------"
+    idx = body.find(literal)
+    if idx >= 0:
+        own = body[:idx].strip()
+        history = body[idx:].strip()
+    else:
+        own = body.strip()
+        history = ""
+    if len(own) > max_own_chars:
+        own = own[:max_own_chars].rstrip() + "\n...[draft body truncated]"
+    if len(history) > max_history_chars:
+        history = history[:max_history_chars].rstrip() + "\n...[quoted history truncated; full history is preserved by Odysseus]"
+    if history:
+        body_out = (
+            f"{own}\n\n" if own else ""
+        ) + (
+            "QUOTED HISTORY EXCERPT FOR CONTEXT ONLY -- do not rewrite or include this excerpt in your tool output; "
+            "Odysseus preserves the full quoted thread below the reply automatically.\n"
+            f"{history}"
+        )
+    else:
+        body_out = own
+    return header.rstrip() + "\n---\n" + body_out.strip()
+
+
 def _minimal_odysseus_doc_messages(messages: List[Dict], active_document, stream_create: bool = False) -> List[Dict]:
     """Tiny prompt path for the Odysseus document LoRA.
 
@@ -1166,6 +1269,10 @@ def _minimal_odysseus_doc_messages(messages: List[Dict], active_document, stream
     else:
         system = (
             "You are Odysseus. Edit or suggest changes to the active document using exactly one fenced tool block when needed.\n"
+            "The active document content is authoritative. Apply the user's request to that content; do not append the user's instruction as document text.\n"
+            "Preserve the current title, language, structure, and existing meaning unless the user explicitly asks to change them.\n"
+            "If the user asks for ALL CAPS/uppercase/lowercase, transform the existing document text itself.\n"
+            "If the user refers to line numbers, use the numbered active document lines; never include the line numbers or tabs in FIND/REPLACE text.\n"
             "If the user asks to add, remove, rewrite, transform, change, capitalize, shorten, expand, or otherwise apply a change, use edit_document or update_document, not suggest_document.\n"
             "Use suggest_document only when the user explicitly asks for suggestions, feedback, or proposed improvements without applying them.\n"
             "For targeted edits:\n"
@@ -1201,20 +1308,98 @@ def _minimal_odysseus_doc_messages(messages: List[Dict], active_document, stream
         out.append(memory_message)
     if active_document is not None:
         content = active_document.current_content or ""
+        if not stream_create:
+            content_for_prompt = "\n".join(
+                f"{idx}\t{line}" for idx, line in enumerate(content.split("\n"), 1)
+            )
+            content_note = (
+                "Content with line numbers. The number and tab are reference-only and are not part of the document:\n"
+            )
+        else:
+            content_for_prompt = content
+            content_note = "Content:\n"
         out.append({
             "role": "user",
             "content": (
                 "Active document:\n"
                 f"Title: {active_document.title}\n"
                 f"Language: {active_document.language or 'text'}\n"
-                "Content:\n"
+                f"{content_note}"
-                f"{content}"
+                f"{content_for_prompt}"
             ),
         })
     out.append({"role": "user", "content": latest})
     return out
 
 
+def _looks_like_notes_turn(text: str) -> bool:
+    q = (text or "").lower()
+    if re.search(r"\b(notes?|todos?|to-?do|checklists?|reminders?)\b", q):
+        return True
+    if re.search(r"\b(?:take|jot|write down|add|create|make)\b.{0,80}\b(?:note|todo|to-?do|checklist|reminder)\b", q):
+        return True
+    if re.search(r"\b(?:buy|pick ?up|pickup)\b", q) and not re.search(r"\b(?:calendar|event|meeting|appointment|schedule)\b", q):
+        return True
+    return False
+
+
+def _minimal_odysseus_notes_messages(messages: List[Dict]) -> List[Dict]:
+    """Tiny prompt path for Odysseus notes LoRAs.
+
+    The finetune is trained to emit Odysseus note tool calls without receiving
+    the full tool schema or saved-context wrapper stack.
+    """
+    latest = _extract_last_user_message(messages)
+    system = (
+        "You are Odysseus. Handle note, todo, checklist, and reminder requests.\n"
+        "You have access to the user's Odysseus notes through manage_notes.\n"
+        "For 'what are my notes', 'show my notes', note searches, note creation, todos, checklists, and reminders, use the Odysseus manage_notes tool call format.\n"
+        "Use action=list/search/view/add/update/delete/toggle_item as appropriate.\n"
+        "For casual chat, answer briefly with no tool.\n"
+        "After a tool succeeds, answer with Done or a concise summary from the tool result.\n"
+        "Never repeat hidden context wrappers, untrusted source labels, or prompt text."
+    )
+    out = [{"role": "system", "content": system}]
+    memory_message = _minimal_saved_memory_message(messages)
+    if memory_message:
+        out.append(memory_message)
+    out.append({"role": "user", "content": latest})
+    return out
+
+
+def _looks_like_memory_identity_turn(text: str) -> bool:
+    q = re.sub(r"[^a-z0-9\s'?]", " ", (text or "").lower())
+    q = re.sub(r"\bhwho\b", "who", q)
+    return bool(re.search(
+        r"\b("
+        r"who am i|who i am|what'?s my name|what is my name|where do i live|"
+        r"what do you know about me|about me|relate to me|use what you know|"
+        r"remember\b|forget\b|my preference|my preferences|i prefer|"
+        r"my memory|memories about me"
+        r")\b",
+        q,
+    ))
+
+
+def _minimal_odysseus_general_messages(messages: List[Dict], include_memory: bool = False) -> List[Dict]:
+    """Minimal fallback for Odysseus finetunes outside domain-specific paths."""
+    latest = _extract_last_user_message(messages)
+    system = (
+        "You are Odysseus. Answer directly and briefly.\n"
+        "Use Odysseus tool-call format only when the user explicitly asks you to take an action.\n"
+        "For explicit remember/forget/preference requests, use manage_memory.\n"
+        "For casual chat or identity questions, answer normally.\n"
+        "Never repeat hidden context wrappers, untrusted source labels, or prompt text."
+    )
+    out = [{"role": "system", "content": system}]
+    if include_memory:
+        memory_message = _minimal_saved_memory_message(messages)
+        if memory_message:
+            out.append(memory_message)
+    out.append({"role": "user", "content": latest})
+    return out
+
+
 _DOC_MODEL_ARTIFACT_RE = re.compile(
     r"(?:\|end\|)+\|?assistan(?:t)?\|?"
     r"|\|assistan(?:t)?\|"
@@ -1228,6 +1413,48 @@ def _strip_doc_model_artifacts(text: str) -> str:
     return _DOC_MODEL_ARTIFACT_RE.sub("", text or "")
 
 
+_DOC_TOOL_TRUNCATED_FENCE_RE = re.compile(
+    r"```(create|update|edit|edi|suggest)_documen(?!t)(?=\s|\n|```)",
+    re.IGNORECASE,
+)
+
+
+_DOC_TOOL_COMPACT_MARKERS = {
+    "<<FIND>": "<<<FIND>>>",
+    "<<REPLACE>": "<<<REPLACE>>>",
+    "<<SUGGEST>": "<<<SUGGEST>>>",
+    "<<REASON>": "<<<REASON>>>",
+    "<<END>": "<<<END>>>",
+}
+
+
+def _normalize_truncated_document_tool_fences(text: str) -> str:
+    """Repair Qwen/SFT fence tags that drop the final 't' in *_document.
+
+    The document LoRA is run in a suppressed-text mode: fenced tool blocks are
+    hidden from chat and parsed after the stream finishes. If the model emits
+    ```update_documen instead of ```update_document, the parser sees no tool and
+    the turn looks like it silently died. Keep this repair scoped to document
+    tool fence tags only.
+    """
+    normalized = _DOC_TOOL_TRUNCATED_FENCE_RE.sub(
+        lambda m: f"```{'edit' if m.group(1).lower() == 'edi' else m.group(1).lower()}_document",
+        text or "",
+    )
+    for compact, full in _DOC_TOOL_COMPACT_MARKERS.items():
+        normalized = normalized.replace(compact, full)
+    marker = r"<<<(?:FIND|REPLACE|SUGGEST|REASON|END)>>>"
+    normalized = re.sub(rf"(?<!\n)({marker})", r"\n\1", normalized)
+    normalized = re.sub(rf"({marker})(?=\S)", r"\1\n", normalized)
+    normalized = re.sub(
+        r"(<<<(?:REPLACE|SUGGEST|REASON)>>>)\n(<<<END>>>)",
+        r"\1\n\n\2",
+        normalized,
+    )
+    normalized = re.sub(r"\n(```)", r"\1", normalized)
+    return normalized
+
+
 def _normalize_stream_document_fences(text: str, target_tool: str = "create_document") -> str:
     """Treat visible ```document/documen blocks as document tool blocks.
 
@@ -1236,7 +1463,9 @@ def _normalize_stream_document_fences(text: str, target_tool: str = "create_docu
     the same shape is a full replacement of the open document, so map it to
     update_document and drop the title/language header lines.
     """
-    text = _strip_doc_model_artifacts(text or "")
+    text = _normalize_truncated_document_tool_fences(
+        _strip_doc_model_artifacts(text or "")
+    )
 
     def repl(match: re.Match) -> str:
         body = match.group(1) or ""
@@ -1387,6 +1616,7 @@ def _build_system_prompt(
     _email_style_message = None
     _integ_message = None
     _mcp_desc_message = None
+    _active_doc_is_email_doc = False
     if active_document:
         set_active_document(active_document.id)
         _doc_raw = active_document.current_content or ""
@@ -1402,19 +1632,23 @@ def _build_system_prompt(
             or _doc_title_l in {"new email", "new mail", "new message"}
             or ("To:" in _doc_raw[:400] and "Subject:" in _doc_raw[:400] and "\n---\n" in _doc_raw)
         )
+        _active_doc_is_email_doc = _is_email_doc
         if _is_email_doc:
+            _email_prompt_doc = _compact_email_draft_context(_doc_raw)
             doc_ctx = (
                 f'ACTIVE EMAIL DRAFT (open in editor — the user is looking at this right now)\n'
                 f'Title: "{active_document.title}"\n'
-                f'```\n{_doc_raw}\n```\n\n'
+                f'```\n{_email_prompt_doc}\n```\n\n'
                 f'This is the current email compose window, not a normal document library item. If the user says "write", "draft", "reply", "make it say", or "write the email" without naming another target, edit THIS email draft.\n\n'
                 f'When the user asks you to write, reply to, or improve this email:\n'
-                f'1. Use `update_document` to replace the ENTIRE content — keep all the header lines (To, Subject, In-Reply-To, References, X-Source-UID, X-Source-Folder, X-Attachments) and the `---` separator EXACTLY as they are.\n'
+                f'1. Use `update_document` to update this email draft — keep all header lines (To, Subject, In-Reply-To, References, X-Source-UID, X-Source-Folder, X-Attachments) and the `---` separator EXACTLY as they are.\n'
-                f'2. Replace ONLY the body text (the part after `---`). If there is a quoted original email (lines starting with `>`), keep that quoted block unchanged BELOW your new reply.\n'
+                f'2. Replace ONLY the new reply text above `---------- Previous message ----------`. You may omit the quoted history from your tool output; Odysseus preserves everything from that separator downward automatically.\n'
                 f'3. Write the reply body above the quoted original. Use the saved email writing style when present.\n'
                 f'4. Identity is critical: write as the logged-in user / mailbox owner only. NEVER sign as the recipient, original sender, quoted sender, spouse, assistant, company, or any third party. If adding a signature, use only the name/signature implied by the saved email writing style.\n'
                 f'5. Mechanical style is critical: never use em dash/en dash; use --. Never use curly apostrophes. For English emails, use Hi/Hiya from the saved style rather than Hey unless the user explicitly asks for Hey.\n'
-                f'6. Do NOT use create_document — the email is already open, you must update it.\n\n'
+                f'6. Do NOT use create_document — the email is already open, you must update it.\n'
+                f'7. Do NOT call read_email/list_emails for this turn. The open email draft above is the source of truth, and the quoted history excerpt is enough context for a reply.\n'
+                f'8. After a successful tool call, answer with a brief confirmation only. Do not paste the full email back into chat unless the user asks.\n\n'
                 f'Do NOT ask the user to paste or share the email — you already have it above.'
             )
         else:
@@ -1528,7 +1762,7 @@ def _build_system_prompt(
     # resolve to the real UID instead of the agent inventing a fresh .md
     # draft with fake headers. This is the email equivalent of _doc_message.
     _email_message = None
-    if active_email and active_email.get("uid"):
+    if active_email and active_email.get("uid") and not _active_doc_is_email_doc:
         _em_uid = active_email.get("uid", "")
         _em_folder = active_email.get("folder", "INBOX")
         _em_account = active_email.get("account", "")
@@ -2328,6 +2562,7 @@ async def stream_agent_loop(
     workspace: Optional[str] = None,
     forced_tools: Optional[Set[str]] = None,
     uploaded_files: Optional[List[Dict]] = None,
+    workload: str = "foreground",
     _is_teacher_run: bool = False,
 ) -> AsyncGenerator[str, None]:
     """Streaming agent loop generator.
@@ -2371,13 +2606,23 @@ async def stream_agent_loop(
     _t0 = time.time()
     _needs_admin = _detect_admin_intent(messages)
     _last_user = _extract_last_user_message(messages)
+    _ody_qwen_finetune_model = (model or "").lower().startswith("odysseus-qwen3")
+    _ody_memory_identity_turn = _looks_like_memory_identity_turn(_last_user)
     _intent = _classify_agent_request(messages, _last_user)
     _low_signal_turn = bool(_intent.get("low_signal"))
     _casual_low_signal_turn = _is_casual_low_signal(_last_user)
+    _existing_conversation = _user_turn_count(messages) > 1
     _active_document_relevant = _turn_targets_active_document(_intent, _last_user, active_document)
+    _active_email_draft_relevant = _active_document_relevant and _is_email_document_obj(active_document)
+    if _active_email_draft_relevant:
+        disabled_tools.update({
+            "list_email_accounts", "list_emails", "read_email",
+            "mcp__email__list_emails", "mcp__email__read_email",
+        })
     _prompt_active_document = active_document if _active_document_relevant else None
     _direct_low_signal = (
         _low_signal_turn
+        and not _existing_conversation
         and not bool(_intent.get("continuation"))
         and not plan_mode
         and not approved_plan
@@ -2400,10 +2645,22 @@ async def stream_agent_loop(
         _active_document_relevant,
         _retrieval_query[:200],
     )
+    if _low_signal_turn and _existing_conversation:
+        logger.info(
+            "[agent] keeping contextual path for low-signal turn in existing conversation latest=%r",
+            _last_user[:80],
+        )
     _mcp_disabled_map = _load_mcp_disabled_map() if mcp_mgr else {}
     if _direct_low_signal:
         logger.info("[agent] direct low-signal reply path for latest=%r", _last_user[:80])
-        direct_messages = [{"role": "user", "content": _last_user}]
+        direct_messages = (
+            _minimal_odysseus_general_messages(
+                messages,
+                include_memory=True,
+            )
+            if _ody_qwen_finetune_model
+            else [{"role": "user", "content": _last_user}]
+        )
         direct_response = ""
         direct_start = time.time()
         direct_actual_model = model
@@ -2419,6 +2676,7 @@ async def stream_agent_loop(
                 tools=None,
                 timeout=int(get_setting("agent_stream_timeout_seconds", 300) or 300),
                 session_id=session_id,
+                workload=workload,
             ):
                 if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
                     try:
@@ -2511,7 +2769,18 @@ async def stream_agent_loop(
     if not guide_only and not _relevant_tools:
         try:
             from src.tool_index import get_tool_index, ALWAYS_AVAILABLE
-            tool_idx = get_tool_index()
+            try:
+                tool_idx = await asyncio.wait_for(
+                    asyncio.to_thread(get_tool_index),
+                    timeout=_TOOL_SELECTION_TIMEOUT_SECONDS,
+                )
+            except asyncio.TimeoutError:
+                logger.warning(
+                    "[tool-rag] Tool index init exceeded %.1fs; falling back to always-available tools",
+                    _TOOL_SELECTION_TIMEOUT_SECONDS,
+                )
+                tool_idx = None
+                _relevant_tools = set(ALWAYS_AVAILABLE)
             if tool_idx:
                 if mcp_mgr:
                     try:
@@ -2532,11 +2801,17 @@ async def stream_agent_loop(
                         )
                         logger.info(f"[tool-rag] Retrieved tools for query: {sorted(_relevant_tools - ALWAYS_AVAILABLE)}")
                     except asyncio.TimeoutError:
+                        # Leave _relevant_tools unset so the keyword fallback
+                        # below still runs. Hard-coding ALWAYS_AVAILABLE here
+                        # skipped the deterministic keyword hints whenever the
+                        # embedding backend was slow (e.g. a remote endpoint
+                        # cold-loading its model), silently stripping email/
+                        # calendar tools from queries that named them outright.
                         logger.warning(
-                            "[tool-rag] Retrieval exceeded %.1fs; falling back to always-available tools",
+                            "[tool-rag] Retrieval exceeded %.1fs; falling back to keyword tool selection",
                             _TOOL_SELECTION_TIMEOUT_SECONDS,
                         )
-                        _relevant_tools = set(ALWAYS_AVAILABLE)
+                        _relevant_tools = None
         except Exception as e:
             logger.warning(f"[tool-rag] Retrieval failed, using keyword fallback: {e}")
             _relevant_tools = None
@@ -2572,7 +2847,13 @@ async def stream_agent_loop(
         if "email" in (_intent.get("domains") or set()):
             _relevant_tools.add("ui_control")
         if "web" in (_intent.get("domains") or set()):
-            _relevant_tools.update({"web_search", "web_fetch"})
+            _relevant_tools.update(WEB_TOOL_NAMES)
+            _blocked_web_tools = sorted(WEB_TOOL_NAMES & disabled_tools)
+            if _blocked_web_tools:
+                logger.info(
+                    "[agent-intent] web domain selected but search tools remain disabled=%s",
+                    _blocked_web_tools,
+                )
         if "ui" in (_intent.get("domains") or set()):
             _relevant_tools.add("ui_control")
 
@@ -2582,6 +2863,19 @@ async def stream_agent_loop(
     # panel is open.
     if _relevant_tools is not None and _active_document_relevant:
         _relevant_tools.update({"edit_document", "update_document", "suggest_document"})
+        if _active_email_draft_relevant:
+            # The open compose document already contains the recipient,
+            # subject, source UID, and quoted previous-message excerpt. Reading
+            # the same email again through IMAP/MCP is slow, token-heavy, and
+            # can hang. Keep draft editing tools, drop email fetch tools.
+            _email_fetch_tools = {
+                "list_email_accounts", "list_emails", "read_email",
+                "mcp__email__list_emails", "mcp__email__read_email",
+            }
+            removed = sorted(_relevant_tools & _email_fetch_tools)
+            if removed:
+                _relevant_tools.difference_update(_email_fetch_tools)
+                logger.info("[agent-intent] active email draft pruned fetch tools=%s", removed)
 
     # Current-turn chat uploads are real files under the upload/data root. Make
     # the read-side file/document tools visible immediately so the agent can
@@ -2592,14 +2886,15 @@ async def stream_agent_loop(
             _relevant_tools = set(ALWAYS_AVAILABLE)
         _relevant_tools.update({"read_file", "grep", "ls", "manage_documents"})
 
-    # Per-request UI toggles are stronger than retrieval. If the user turns on
+    # Per-request forced tools are stronger than retrieval. Explicit search
-    # Search, the model must see the search tools even when the latest text is a
+    # settings make web tools visible even when tool RAG misses them;
-    # typo or otherwise low-signal for tool RAG.
+    # route-level disabled_tools decides what remains allowed.
     if not guide_only and forced_tools:
+        forced_set = {t for t in forced_tools if t not in disabled_tools}
         if _relevant_tools is None:
             from src.tool_index import ALWAYS_AVAILABLE
             _relevant_tools = set(ALWAYS_AVAILABLE)
-        _relevant_tools.update(t for t in forced_tools if t not in disabled_tools)
+        _relevant_tools.update(forced_set)
 
     # The skill index injected by _build_system_prompt tells the model to
     # call `manage_skills action=view`, and Jaccard-matched skills are pasted
@@ -2641,7 +2936,7 @@ async def stream_agent_loop(
 
     _intent_domains = set(_intent.get("domains") or set())
     _ody_doc_finetune_mode = (
-        (model or "").lower().startswith("odysseus-qwen3")
+        _ody_qwen_finetune_model
         and (
             "documents" in _intent_domains
             or _active_document_relevant
@@ -2650,6 +2945,14 @@ async def stream_agent_loop(
         and "files" not in _intent_domains
         and not guide_only
     )
+    _ody_notes_finetune_mode = (
+        _ody_qwen_finetune_model
+        and not _ody_doc_finetune_mode
+        and ("notes_calendar_tasks" in _intent_domains or _looks_like_notes_turn(_last_user))
+        and _looks_like_notes_turn(_last_user)
+        and "files" not in _intent_domains
+        and not guide_only
+    )
     _ody_doc_stream_create_mode = _ody_doc_finetune_mode and _prompt_active_document is None
     if _ody_doc_finetune_mode and _relevant_tools is not None:
         if _prompt_active_document is not None:
@@ -2660,6 +2963,36 @@ async def stream_agent_loop(
         else:
             _relevant_tools = {"create_document", "ask_user", "update_plan"}
         logger.info("[agent-intent] odysseus doc finetune tool clamp=%s", sorted(_relevant_tools))
+    elif _ody_notes_finetune_mode and _relevant_tools is not None:
+        _relevant_tools = {"manage_notes", "ask_user", "update_plan"}
+        logger.info("[agent-intent] odysseus notes finetune tool clamp=%s", sorted(_relevant_tools))
+
+    if (
+        _relevant_tools is not None
+        and _active_document_relevant
+        and "files" not in _intent_domains
+        and not uploaded_files
+        and not workspace
+    ):
+        _doc_irrelevant_file_tools = {
+            "append_file",
+            "bash",
+            "edit_file",
+            "glob",
+            "grep",
+            "ls",
+            "read_file",
+            "replace_file",
+            "run_shell",
+            "write_file",
+        }
+        _removed_doc_file_tools = sorted(_relevant_tools & _doc_irrelevant_file_tools)
+        if _removed_doc_file_tools:
+            _relevant_tools.difference_update(_doc_irrelevant_file_tools)
+            logger.info(
+                "[agent-intent] active document turn removed file tools=%s",
+                _removed_doc_file_tools,
+            )
 
     if _relevant_tools is not None:
         logger.info("[agent-intent] selected_tools=%s", sorted(_relevant_tools)[:50])
@@ -2761,6 +3094,24 @@ async def stream_agent_loop(
             _ody_doc_stream_create_mode,
             len(messages),
         )
+    elif _ody_notes_finetune_mode and not plan_mode and not approved_plan and not guide_only:
+        messages = _minimal_odysseus_notes_messages(messages)
+        mcp_schemas = []
+        logger.info(
+            "[agent-intent] odysseus notes minimal prompt active messages=%s",
+            len(messages),
+        )
+    elif _ody_qwen_finetune_model and not plan_mode and not approved_plan and not guide_only:
+        messages = _minimal_odysseus_general_messages(
+            messages,
+            include_memory=True,
+        )
+        mcp_schemas = []
+        logger.info(
+            "[agent-intent] odysseus general minimal prompt active include_memory=%s messages=%s",
+            _ody_memory_identity_turn,
+            len(messages),
+        )
     if plan_mode and not guide_only:
         # Steer the model to investigate-then-propose. Hard tool gating handles
         # every write path except shell; this directive is what keeps the
@@ -2875,6 +3226,7 @@ async def stream_agent_loop(
     requested_model = model
     actual_model = model
     total_tool_calls = 0  # for budget enforcement
+    _ody_notes_tool_completed = False
 
     # Loop-breaker state. Small models (e.g. deepseek-v4-flash) can get
     # stuck firing the same tool call over and over with no text — burns
@@ -2972,7 +3324,7 @@ async def stream_agent_loop(
                     if s.get("function", {}).get("name") not in _ADMIN_SCHEMA_NAMES
                 ]
                 all_tool_schemas = base_schemas + mcp_schemas
-            if _ody_doc_finetune_mode:
+            if _ody_qwen_finetune_model:
                 all_tool_schemas = []
             if disabled_tools:
                 all_tool_schemas = [
@@ -3022,6 +3374,7 @@ async def stream_agent_loop(
             tool_choice_none=_ody_doc_finetune_mode,
             timeout=agent_stream_timeout,
             session_id=session_id,
+            workload=workload,
         ):
             if not _round_first_event_logged:
                 _round_first_event_logged = True
@@ -3143,11 +3496,15 @@ async def stream_agent_loop(
                         if data.get("thinking"):
                             round_reasoning += data["delta"]
                         else:
-                            _delta_text = _strip_doc_model_artifacts(data["delta"]) if _ody_doc_finetune_mode else data["delta"]
+                            _delta_text = (
+                                _strip_doc_model_artifacts(data["delta"])
+                                if _ody_qwen_finetune_model
+                                else data["delta"]
+                            )
                             round_response += _delta_text
                             full_response += _delta_text
                             data["delta"] = _delta_text
-                        if not _ody_doc_finetune_mode or data.get("thinking"):
+                        if not _ody_qwen_finetune_model or data.get("thinking"):
                             yield f"data: {json.dumps(data)}\n\n"
                         # Detect text-fence doc streaming. Normal agent prompts
                         # use ```create_document; the doc LoRA streaming path
@@ -3268,6 +3625,59 @@ async def stream_agent_loop(
                     else converted_calls[:1]
                 )
 
+        if _ody_qwen_finetune_model and tool_blocks:
+            _allowed_memory_write_actions = {"add", "edit", "update", "delete", "delete_all"}
+            _explicit_memory_browse = bool(re.search(
+                r"\b(search|list|show|open|view)\b.{0,40}\b(memories|memory|brain)\b",
+                _last_user.lower(),
+            ))
+            _filtered_tool_blocks = []
+            _filtered_converted_calls = []
+            _dropped_memory_lookup = False
+            for _idx, _block in enumerate(tool_blocks):
+                if _block.tool_type != "manage_memory":
+                    _filtered_tool_blocks.append(_block)
+                    if _idx < len(converted_calls):
+                        _filtered_converted_calls.append(converted_calls[_idx])
+                    continue
+                _action = ""
+                try:
+                    _args = json.loads(_block.content or "{}")
+                    if isinstance(_args, dict):
+                        _action = str(_args.get("action") or "").lower()
+                except Exception:
+                    _action = ""
+                if _action in {"list", "search", "view", "get", "read"} and not _explicit_memory_browse:
+                    _dropped_memory_lookup = True
+                elif _action in _allowed_memory_write_actions and re.search(
+                    r"\b(remember|forget|preference|prefer|save this about me|update memory|delete memory)\b",
+                    _last_user.lower(),
+                ):
+                    _filtered_tool_blocks.append(_block)
+                    if _idx < len(converted_calls):
+                        _filtered_converted_calls.append(converted_calls[_idx])
+                else:
+                    _dropped_memory_lookup = True
+            if _dropped_memory_lookup:
+                logger.info(
+                    "[agent-intent] odysseus qwen dropped manage_memory lookup; answering from compact memory"
+                )
+                tool_blocks = _filtered_tool_blocks
+                converted_calls = _filtered_converted_calls
+                if used_native:
+                    native_tool_calls = _filtered_converted_calls
+                if not tool_blocks:
+                    _force_answer = True
+                    messages.append({
+                        "role": "system",
+                        "content": (
+                            "Answer the user's identity/personal-memory question from the compact "
+                            "saved memory facts already provided. Do not call manage_memory or any tool."
+                        ),
+                    })
+                    yield f'data: {json.dumps({"type": "agent_step", "round": round_num + 1})}\n\n'
+                    continue
+
         # Force-answer round: we told the model to STOP calling tools and
         # answer. If it ignored that and emitted a (possibly DSML) tool
         # call anyway, discard it — don't execute, don't re-loop. Keep
@@ -3352,6 +3762,8 @@ async def stream_agent_loop(
         # on reload (#3222 follow-up).
         cleaned_round = strip_tool_blocks(round_response, skip_fenced=(_is_api_model and not used_native and not guide_only)).strip()
         round_texts.append(cleaned_round)
+        if _ody_qwen_finetune_model and not tool_blocks and cleaned_round:
+            yield f'data: {json.dumps({"delta": cleaned_round})}\n\n'
 
         if not tool_blocks:
             # ── Completion verifier (mechanism 3a) ────────────────────
@@ -3416,9 +3828,8 @@ async def stream_agent_loop(
                 and _intent_match is not None
                 and len(_intent_text) < 400
                 and "```" not in _intent_text
-                and _intent_nudge_count < _MAX_INTENT_NUDGES
             )
-            if _looks_like_promise:
+            if _looks_like_promise and _intent_nudge_count < _MAX_INTENT_NUDGES:
                 _intent_nudge_count += 1
                 _matched_phrase = _intent_match.group(0).strip()
                 logger.info(f"[agent] intent-without-action nudge #{_intent_nudge_count} on round {round_num}: {_matched_phrase!r}")
@@ -3447,6 +3858,31 @@ async def stream_agent_loop(
                 # Visible signal in the stream so the user knows we caught it.
                 yield f'data: {json.dumps({"type": "agent_step", "round": round_num + 1})}\n\n'
                 continue
+            if _looks_like_promise:
+                _matched_phrase = _intent_match.group(0).strip()
+                _guard_message = (
+                    "The agent stopped because it repeatedly announced a tool "
+                    "action without making the tool call."
+                )
+                logger.warning(
+                    "[agent] intent-without-action guard exhausted on round %d after %d nudges: %r",
+                    round_num,
+                    _intent_nudge_count,
+                    _matched_phrase,
+                )
+                yield (
+                    "data: "
+                    + json.dumps({
+                        "type": "intent_nudge_exhausted",
+                        "reason": "intent_without_action_nudge_cap",
+                        "message": _guard_message,
+                        "round": round_num,
+                        "nudges": _intent_nudge_count,
+                        "matched": _matched_phrase,
+                    })
+                    + "\n\n"
+                )
+                break
             break  # no tools — done
 
         # ── Loop-breaker (Terminus-style stall detector) ──────────────
@@ -3483,6 +3919,21 @@ async def stream_agent_loop(
             reason = (f"calling {_runaway} with identical arguments over and over" if _runaway
                       else "repeating the same tool calls without new progress")
             logger.warning(f"[agent] loop-breaker tripped on round {round_num} ({reason}); sig={_sig[:80]!r}")
+            yield (
+                "data: "
+                    + json.dumps({
+                    "type": "loop_breaker_triggered",
+                    "reason": "loop_breaker_stall",
+                    "message": (
+                        "The loop-breaker detected repeated tool calls without "
+                        "new progress, so the agent is being forced to stop "
+                        "using tools and give its best final answer."
+                    ),
+                    "round": round_num,
+                    "detail": reason,
+                })
+                + "\n\n"
+            )
             # The model has been executing tools, so its results are already
             # in context. Force ONE tool-free round to converge: write the
             # answer from what it has, or state plainly what's blocking it.
@@ -3601,16 +4052,31 @@ async def stream_agent_loop(
                         await _progress_q.put(None)
 
                 _tool_task = asyncio.create_task(_run_tool())
-                # Drain progress events as they arrive — block until the
+                try:
-                # next event OR the tool finishes (sentinel = None).
+                    # Drain progress events as they arrive — block until the
-                while True:
+                    # next event OR the tool finishes (sentinel = None).
-                    evt = await _progress_q.get()
+                    while True:
-                    if evt is None:
+                        evt = await _progress_q.get()
-                        break
+                        if evt is None:
-                    yield (
+                            break
-                        f'data: {json.dumps({"type": "tool_progress", "tool": block.tool_type, "round": round_num, **evt})}\n\n'
+                        yield (
-                    )
+                            f'data: {json.dumps({"type": "tool_progress", "tool": block.tool_type, "round": round_num, **evt})}\n\n'
-                desc, result = await _tool_task
+                        )
+                    desc, result = await _tool_task
+                finally:
+                    # If the SSE client disconnects (or this generator is
+                    # otherwise closed) while we're awaiting a progress event
+                    # above, GeneratorExit is thrown in right here and the
+                    # `await _tool_task` on the line above never runs — the
+                    # task (and any subprocess execute_tool_block spawned for
+                    # bash/python tools) would otherwise keep running
+                    # orphaned with nothing left to await or cancel it.
+                    if not _tool_task.done():
+                        _tool_task.cancel()
+                        try:
+                            await _tool_task
+                        except (asyncio.CancelledError, Exception):
+                            pass
 
             # A skill the model just loaded can prescribe tools that weren't
             # RAG-selected this turn (declared via requires_toolsets in its
@@ -3813,6 +4279,39 @@ async def stream_agent_loop(
                 tool_output_data["diff"] = result["diff"]
             yield f'data: {json.dumps(tool_output_data)}\n\n'
 
+            if block.tool_type == "manage_notes":
+                _notes_action = ""
+                try:
+                    _notes_args = json.loads(block.content or "{}")
+                    if isinstance(_notes_args, dict):
+                        _notes_action = str(_notes_args.get("action") or "").lower()
+                except Exception:
+                    _notes_action = ""
+                _notes_text = ""
+                if not result.get("error"):
+                    if _notes_action in {"list", "search", "find", "view", "lis"}:
+                        _notes_text = _note_list_summary_from_tool_output(
+                            result.get("output") or result.get("results") or result.get("content") or ""
+                        )
+                    elif _notes_action in {"add", "update", "delete", "toggle_item"}:
+                        _notes_text = str(
+                            result.get("response")
+                            or result.get("output")
+                            or result.get("results")
+                            or ""
+                        ).strip()
+                        if _notes_text.startswith("AI: "):
+                            _notes_text = _notes_text[4:].strip()
+                        if _notes_text and not re.match(r"^(done|note|item|deleted)\b", _notes_text, re.IGNORECASE):
+                            _notes_text = f"Done — {_notes_text}"
+                if _notes_text:
+                    _clean_current = strip_tool_blocks(full_response).strip()
+                    if _notes_text not in _clean_current:
+                        _prefix = "\n\n" if _clean_current else ""
+                        full_response = (_clean_current + _prefix + _notes_text).strip()
+                        yield f'data: {json.dumps({"delta": _prefix + _notes_text})}\n\n'
+                    _ody_notes_tool_completed = True
+
             # This must be the final UI event for ask_user: the frontend appends
             # the card below the now-settled tool node and cancels any between-
             # round spinner.  The turn ends after the current tool batch.
@@ -3857,6 +4356,7 @@ async def stream_agent_loop(
                 _title = (result.get("note_title") or "").strip()
                 _label = f"View note: {_title}" if _title else "View note"
                 _anchor = f"\n\n[{_label}](#note-{_nid})\n"
+                full_response = (full_response.rstrip() + _anchor).strip()
                 yield 'data: ' + json.dumps({"delta": _anchor}) + '\n\n'
 
             # Save for history persistence
@@ -3928,6 +4428,10 @@ async def stream_agent_loop(
             logger.info("[agent] odysseus doc tool completed after one textual tool block")
             break
 
+        if _ody_notes_finetune_mode and _ody_notes_tool_completed:
+            logger.info("[agent] odysseus notes completed from deterministic tool output")
+            break
+
         # Feed results back to LLM for next round
         # Pass the CONVERTED calls (aligned 1:1 with tool_result_texts), not the
         # raw native_tool_calls: a call that failed to convert is dropped from
@@ -3968,6 +4472,27 @@ async def stream_agent_loop(
     if _fallback_chunk:
         yield _fallback_chunk
 
+    # Do not persist raw textual tool-call JSON / role markers as assistant
+    # prose. Local finetunes may emit those before the parser catches and
+    # executes them; saved history should contain only the user-facing answer.
+    full_response = strip_tool_blocks(full_response).strip()
+    if _ody_notes_finetune_mode and tool_events:
+        for _ev in reversed(tool_events):
+            if _ev.get("tool") != "manage_notes":
+                continue
+            _notes_action = ""
+            try:
+                _cmd_args = json.loads(_ev.get("command") or "{}")
+                if isinstance(_cmd_args, dict):
+                    _notes_action = str(_cmd_args.get("action") or "").lower()
+            except Exception:
+                _notes_action = ""
+            if _notes_action in {"list", "search", "find", "view", "lis"}:
+                _notes_summary = _note_list_summary_from_tool_output(_ev.get("output") or "")
+                if _notes_summary:
+                    full_response = _notes_summary
+                break
+
     # --- Final metrics ---
     total_duration = time.time() - total_start
     metrics = _compute_final_metrics(

src/agent_tools/document_tools.py

@@ -2,10 +2,16 @@ from typing import Any, Dict, List, Optional
 import logging
 import re
 from src.constants import MAX_READ_CHARS
-from src.tool_utils import _parse_tool_args
+from src.tool_utils import _parse_tool_args, get_upload_handler
+from src.upload_handler import reserve_upload_references
 
 logger = logging.getLogger(__name__)
 
+
+def _missing_document_upload(owner: Optional[str], content: Any) -> Optional[str]:
+    """Reserve explicit upload URLs before an agent persists document text."""
+    return reserve_upload_references(get_upload_handler(), owner, content)
+
 # ---------------------------------------------------------------------------
 # Active document state
 # ---------------------------------------------------------------------------
@@ -130,15 +136,70 @@ def _looks_like_email_document(text: str = "", title: str = "") -> bool:
         return True
     return bool(_re.search(r"(?im)^To:\s*", s) and _re.search(r"(?im)^Subject:\s*", s))
 
+def _split_email_header_body(text: str) -> tuple[str, str]:
+    if "\n---\n" in (text or ""):
+        header, body = (text or "").split("\n---\n", 1)
+        return header.rstrip(), body.strip()
+    return (text or "").strip(), ""
+
+def _split_email_reply_history(body: str) -> tuple[str, str]:
+    """Split draft body from quoted/original email history.
+
+    Email reply docs keep the original thread below the user's new reply. Models
+    often rewrite only the fresh reply body; this helper keeps the historical
+    block from being wiped when update_document/edit_document replaces content.
+    """
+    text = body or ""
+    literal = "---------- Previous message ----------"
+    literal_idx = text.find(literal)
+    if literal_idx >= 0:
+        return text[:literal_idx].strip(), text[literal_idx:].strip()
+    patterns = [
+        r"(?m)^On .+ wrote:\s*$",
+        r"(?m)^> .+",
+    ]
+    starts = []
+    for pat in patterns:
+        m = re.search(pat, text)
+        if m:
+            starts.append(m.start())
+    if not starts:
+        return text.strip(), ""
+    idx = min(starts)
+    return text[:idx].strip(), text[idx:].strip()
+
+def _merge_email_headers(old_header: str, new_header: str) -> str:
+    """Preserve routing/threading metadata if a model omits it."""
+    protected = (
+        "In-Reply-To", "References", "X-Source-UID", "X-Source-Folder",
+        "X-Attachments", "X-Forward-Attachments",
+    )
+    lines = [l for l in (new_header or "").splitlines() if l.strip()]
+    present = {l.split(":", 1)[0].strip().lower() for l in lines if ":" in l}
+    for old_line in (old_header or "").splitlines():
+        if ":" not in old_line:
+            continue
+        key = old_line.split(":", 1)[0].strip()
+        if key in protected and key.lower() not in present:
+            lines.append(old_line)
+            present.add(key.lower())
+    return "\n".join(lines).rstrip()
+
 def _coerce_email_document_content(existing: str, incoming: str) -> str:
     """Keep email docs in the To/Subject/---/body shape even if a model writes
     only the body or dumps header labels without the separator."""
     import re as _re
     old = existing or ""
     new = (incoming or "").strip()
+    old_header, old_body = _split_email_header_body(old)
+    _, old_history = _split_email_reply_history(old_body)
     if "\n---\n" in new:
-        return new
+        new_header, new_body = _split_email_header_body(new)
-    header = old.split("\n---\n", 1)[0] if "\n---\n" in old else "To: \nSubject: "
+        new_own, new_history = _split_email_reply_history(new_body)
+        if old_history and not new_history:
+            new_body = (new_own + "\n\n" + old_history).strip()
+        return _merge_email_headers(old_header, new_header).rstrip() + "\n---\n" + new_body
+    header = old_header if old_header else "To: \nSubject: "
     if _looks_like_email_document(new):
         lines = new.splitlines()
         last_header_idx = -1
@@ -152,6 +213,9 @@ def _coerce_email_document_content(existing: str, incoming: str) -> str:
         body = "\n".join(body_lines).strip()
     else:
         body = new
+    _, incoming_history = _split_email_reply_history(body)
+    if old_history and not incoming_history:
+        body = (body.strip() + "\n\n" + old_history).strip()
     return header.rstrip() + "\n---\n" + body
 
 def parse_edit_blocks(content: str) -> list:
@@ -326,6 +390,13 @@ class CreateDocumentTool:
                 return {"error": "Cannot create document in another user's session"}
             _owner = _sess.owner if _sess else None
 
+            missing_id = _missing_document_upload(_owner, content)
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
+
             doc = Document(
                 id=doc_id,
                 session_id=session_id,
@@ -397,6 +468,13 @@ class UpdateDocumentTool:
             if is_email_doc:
                 doc.language = "email"
 
+            missing_id = _missing_document_upload(owner, new_content)
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
+
             if not is_email_doc and _pdf_source_upload_id(doc.current_content or ""):
                 return _create_pdf_text_derivative(
                     db,
@@ -463,6 +541,48 @@ class EditDocumentTool:
             if not doc:
                 return {"error": "No documents exist to edit"}
 
+            is_email_doc = doc.language == "email" or _looks_like_email_document(doc.current_content or "", doc.title or "")
+            blank_find_edits = [e for e in edits if not (e.get("find") or "").strip()]
+            if blank_find_edits:
+                if is_email_doc:
+                    replacement_body = (blank_find_edits[0].get("replace") or "").strip()
+                    if not replacement_body:
+                        return {"error": "No edits applied — blank FIND block had no replacement text"}
+                    updated_content = _coerce_email_document_content(doc.current_content or "", replacement_body)
+                    applied = 1
+                    skipped = max(0, len(edits) - 1)
+                    doc.language = "email"
+                    missing_id = _missing_document_upload(owner, updated_content)
+                    if missing_id:
+                        return {
+                            "error": f"Referenced upload is no longer available: {missing_id}",
+                            "exit_code": 1,
+                        }
+                    new_ver = doc.version_count + 1
+                    ver = DocumentVersion(
+                        id=str(uuid.uuid4()),
+                        document_id=target_id,
+                        version_number=new_ver,
+                        content=updated_content,
+                        summary=f"Edited email body by {_active_model or 'AI'}",
+                        source="ai",
+                    )
+                    doc.current_content = updated_content
+                    doc.version_count = new_ver
+                    db.add(ver)
+                    db.commit()
+                    return {
+                        "action": "edit",
+                        "doc_id": target_id,
+                        "title": doc.title,
+                        "language": doc.language,
+                        "content": updated_content,
+                        "version": new_ver,
+                        "applied": applied,
+                        "skipped": skipped,
+                    }
+                return {"error": "No edits applied — FIND text cannot be blank"}
+
             updated_content = doc.current_content
             applied = 0
             skipped = 0
@@ -490,6 +610,13 @@ class EditDocumentTool:
             if applied == 0:
                 return {"error": f"No edits applied — none of the FIND blocks matched the document content (skipped {skipped})"}
 
+            missing_id = _missing_document_upload(owner, updated_content)
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
+
             if _pdf_source_upload_id(doc.current_content or ""):
                 return _create_pdf_text_derivative(
                     db,

src/agent_tools/filesystem_tools.py

@@ -424,8 +424,12 @@ class GrepTool:
                     cmd.append("--ignore-case")
                 if glob_pat:
                     cmd += ["--glob", glob_pat]
+                # --iglob (not --glob) so the exclusion is case-insensitive:
+                # on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
+                # resolve to the same secret as their lowercase forms, and the
+                # Python fallback below already folds case via _is_sensitive_path.
                 for _pat in _SENSITIVE_FILE_PATTERNS:
-                    cmd += ["--glob", f"!*{_pat}*"]
+                    cmd += ["--iglob", f"!*{_pat}*"]
                 for _d in _CODENAV_SKIP_DIRS:
                     cmd += ["--glob", f"!**/{_d}/**"]
                 cmd += ["--regexp", pattern, root]

src/agent_tools/session_tools.py

@@ -184,8 +184,12 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner:
     if not sess:
         return {"error": f"Session '{target_sid}' not found"}
 
-    # Owner-scope: reject access to another user's session
+    # Owner-scope: reject access to another user's session. When the caller is
-    if owner and getattr(sess, "owner", None) and sess.owner != owner:
+    # authenticated, a null-owner (legacy / auth-was-off) session is not theirs
+    # either — list_sessions (get_sessions_for_user) and manage_session already
+    # exclude those, so treating it as reachable here let an authenticated agent
+    # read/write a session the other tools hide. Require an exact owner match.
+    if owner and getattr(sess, "owner", None) != owner:
         return {"error": f"Session '{target_sid}' not found"}
 
     if not message:

src/agent_tools/web_tools.py

@@ -1,4 +1,5 @@
 import asyncio
+import inspect
 import json
 from typing import Dict, Any
 
@@ -109,8 +110,20 @@ class WebFetchTool:
             url = "https://" + url
         loop = asyncio.get_running_loop()
         try:
+            def _fetch():
+                kwargs = {"timeout": 10}
+                try:
+                    sig = inspect.signature(fetch_webpage_content)
+                    if "max_bytes" in sig.parameters:
+                        kwargs["max_bytes"] = max_bytes
+                except (TypeError, ValueError):
+                    # Some deployed/test shims may not expose a signature.
+                    # Prefer compatibility over failing the whole fetch.
+                    pass
+                return fetch_webpage_content(url, **kwargs)
+
             result = await asyncio.wait_for(
-                loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10, max_bytes=max_bytes)),
+                loop.run_in_executor(None, _fetch),
                 timeout=30,
             )
         except asyncio.TimeoutError:

src/app_initializer.py

@@ -21,6 +21,7 @@ from src.model_discovery import ModelDiscovery
 from src.chat_handler import ChatHandler
 from src.research_handler import ResearchHandler
 from src.upload_handler import UploadHandler
+from src.tool_utils import set_upload_handler
 from src.search import update_search_config
 
 logger = logging.getLogger(__name__)
@@ -49,6 +50,8 @@ def initialize_managers(base_dir: str, rag_manager=None) -> Dict[str, Any]:
     session_manager = SessionManager(SESSIONS_FILE)
     set_session_manager(session_manager)  # Enable Session.add_message() persistence
     upload_handler = UploadHandler(base_dir, UPLOAD_DIR)
+    session_manager.upload_handler = upload_handler
+    set_upload_handler(upload_handler)
     personal_docs_manager = PersonalDocsManager(PERSONAL_DIR, rag_manager)
     api_key_manager = APIKeyManager(DATA_DIR)
     preset_manager = PresetManager(DATA_DIR)

src/attachment_refs.py

@@ -0,0 +1,164 @@
+"""Attachment reference helpers for chat storage and tool manifests.
+
+Live model calls may need provider-specific data URLs for the current turn.
+Persisted history and search indexes should keep stable upload references and
+human-readable text instead of duplicating raw media bytes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Iterable
+
+
+DATA_URL_RE = re.compile(
+    r"data:[^;,\s\"']+;base64,[A-Za-z0-9+/=]+",
+    re.IGNORECASE,
+)
+
+MEDIA_BLOCK_TYPES = {
+    "image",
+    "image_url",
+    "input_image",
+    "audio",
+    "input_audio",
+    "file",
+}
+
+
+def strip_inline_data_urls(text: str) -> str:
+    """Replace inline data URLs with a compact marker."""
+    if not isinstance(text, str) or ";base64," not in text:
+        return text
+    return DATA_URL_RE.sub("[inline media omitted from persisted history]", text)
+
+
+def attachment_ref(info: dict[str, Any]) -> dict[str, Any]:
+    """Return the stable attachment reference shape used outside raw uploads."""
+    upload_id = str(info.get("id") or info.get("attachment_id") or "").strip()
+    try:
+        size = int(info.get("size") or 0)
+    except (TypeError, ValueError):
+        size = 0
+    ref = {
+        "type": "attachment_ref",
+        "attachment_id": upload_id,
+        "name": info.get("name") or info.get("original_name") or upload_id,
+        "mime": info.get("mime") or "application/octet-stream",
+        "size": size,
+    }
+    checksum = info.get("checksum_sha256") or info.get("sha256") or info.get("hash")
+    if checksum:
+        ref["checksum_sha256"] = checksum
+    created_at = info.get("created_at") or info.get("uploaded_at")
+    if created_at:
+        ref["created_at"] = created_at
+    for key in ("width", "height", "vision", "vision_model", "gallery_id"):
+        value = info.get(key)
+        if value is not None:
+            ref[key] = value
+    return ref
+
+
+def attachment_refs_from_metadata(metadata: dict[str, Any] | None) -> list[dict[str, Any]]:
+    """Extract attachment refs from message metadata."""
+    attachments = (metadata or {}).get("attachments") or []
+    if not isinstance(attachments, list):
+        return []
+    refs: list[dict[str, Any]] = []
+    for item in attachments:
+        if isinstance(item, dict):
+            ref = attachment_ref(item)
+            if ref.get("attachment_id"):
+                refs.append(ref)
+    return refs
+
+
+def _ref_line(ref: dict[str, Any]) -> str:
+    parts = [f"Attachment: {ref.get('name') or ref.get('attachment_id') or 'upload'}"]
+    if ref.get("attachment_id"):
+        parts.append(f"id={ref['attachment_id']}")
+    if ref.get("mime"):
+        parts.append(f"mime={ref['mime']}")
+    if ref.get("size"):
+        parts.append(f"size={ref['size']} bytes")
+    if ref.get("checksum_sha256"):
+        parts.append(f"sha256={ref['checksum_sha256']}")
+    line = "[" + " | ".join(parts) + "]"
+    if ref.get("vision"):
+        line += f"\n[Attachment description: {str(ref['vision']).strip()}]"
+    return line
+
+
+def _text_from_blocks(blocks: Iterable[Any]) -> str:
+    lines: list[str] = []
+    omitted_media = 0
+    for block in blocks:
+        if isinstance(block, str):
+            lines.append(strip_inline_data_urls(block))
+            continue
+        if not isinstance(block, dict):
+            continue
+        block_type = block.get("type")
+        if block_type == "text":
+            text = block.get("text")
+            if isinstance(text, str) and text:
+                lines.append(strip_inline_data_urls(text))
+        elif block_type == "attachment_ref":
+            lines.append(_ref_line(block))
+        elif block_type in MEDIA_BLOCK_TYPES:
+            omitted_media += 1
+        else:
+            try:
+                encoded = json.dumps(block, ensure_ascii=True, sort_keys=True)
+            except TypeError:
+                encoded = str(block)
+            lines.append(strip_inline_data_urls(encoded))
+    if omitted_media:
+        plural = "s" if omitted_media != 1 else ""
+        lines.append(f"[{omitted_media} inline media payload{plural} omitted]")
+    return "\n".join(line for line in lines if line).strip()
+
+
+def persistable_message_content(
+    content: Any,
+    metadata: dict[str, Any] | None = None,
+) -> str:
+    """Return content safe for DB persistence and FTS indexing.
+
+    Multimodal provider blocks are collapsed to readable text plus stable
+    attachment reference lines from metadata. This avoids storing base64 media
+    in ``chat_messages.content`` while preserving enough context for reloads,
+    search, and later turns.
+    """
+    if isinstance(content, list):
+        text = _text_from_blocks(content)
+        refs = attachment_refs_from_metadata(metadata)
+        ref_lines = [_ref_line(ref) for ref in refs]
+        if ref_lines:
+            text = "\n".join([part for part in (text, "\n".join(ref_lines)) if part]).strip()
+        return text
+    if isinstance(content, str):
+        return strip_inline_data_urls(content)
+    try:
+        return strip_inline_data_urls(json.dumps(content, ensure_ascii=True, sort_keys=True))
+    except TypeError:
+        return strip_inline_data_urls(str(content))
+
+
+def search_index_text(content: Any) -> str:
+    """Best-effort searchable text for legacy stored content."""
+    if isinstance(content, str):
+        raw = content.strip()
+        if raw.startswith("[") and '"type"' in raw:
+            try:
+                parsed = json.loads(content)
+            except (TypeError, ValueError):
+                parsed = None
+            if isinstance(parsed, list):
+                return _text_from_blocks(parsed)
+        return strip_inline_data_urls(content)
+    if isinstance(content, list):
+        return _text_from_blocks(content)
+    return persistable_message_content(content)

src/builtin_actions.py

@@ -1125,14 +1125,14 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo
             conn = _imap_connect(None, owner=owner)
             try:
                 conn.select("INBOX", readonly=True)
-                status, data = conn.search(None, "ALL")
+                status, data = conn.uid("SEARCH", None, "ALL")
                 if status != "OK" or not data or not data[0]:
                     return results
                 uids = data[0].split()[-300:][::-1]  # newest 300
                 for uid in uids:
                     try:
-                        st, msg_data = conn.fetch(
+                        st, msg_data = conn.uid(
-                            uid, "(BODY.PEEK[HEADER.FIELDS (FROM)])"
+                            "FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM)])"
                         )
                         if st != "OK" or not msg_data or not msg_data[0]:
                             continue
@@ -1214,7 +1214,7 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo
                     conn2.select("INBOX", readonly=True)
                     for mm in _msgs:
                         try:
-                            st, data = conn2.fetch(mm["uid"], "(BODY.PEEK[TEXT])")
+                            st, data = conn2.uid("FETCH", mm["uid"], "(BODY.PEEK[TEXT])")
                             if st != "OK" or not data or not data[0]:
                                 continue
                             raw = data[0][1] if isinstance(data[0], tuple) else None
@@ -1356,13 +1356,13 @@ async def action_daily_brief(owner: str, **kwargs) -> Tuple[str, bool]:
             conn = _imap_connect(None)
             try:
                 conn.select("INBOX", readonly=True)
-                status, data = conn.search(None, "UNSEEN")
+                status, data = conn.uid("SEARCH", None, "UNSEEN")
                 uids = (data[0].split() if status == "OK" and data and data[0] else [])
                 unread_count = len(uids)
                 # Grab headers for the most recent 5 unread (UIDs increase with arrival)
                 for uid in uids[-5:][::-1]:
                     try:
-                        _, msg_data = conn.fetch(uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT)])")
+                        _, msg_data = conn.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT)])")
                         if not msg_data or not msg_data[0]:
                             continue
                         hdr = msg_data[0][1] if isinstance(msg_data[0], tuple) else msg_data[0]

src/builtin_mcp.py

@@ -104,6 +104,21 @@ def _spawn_bg(coro) -> asyncio.Task:
     return task
 
 
+def builtin_python_env(base_dir: str) -> dict[str, str]:
+    """Environment for built-in Python MCP subprocesses.
+
+    The app root must be importable so mcp_servers can import local modules, but
+    replacing PYTHONPATH entirely hides site-packages in container/dev launches
+    that rely on PYTHONPATH for their active environment.
+    """
+    existing = os.environ.get("PYTHONPATH", "")
+    parts = [base_dir]
+    for item in existing.split(os.pathsep):
+        if item and item not in parts:
+            parts.append(item)
+    return {"PYTHONPATH": os.pathsep.join(parts)}
+
+
 async def register_builtin_servers(mcp_manager):
     """Connect all built-in MCP servers to the manager."""
     if MCP_DISABLED:
@@ -121,7 +136,7 @@ async def register_builtin_servers(mcp_manager):
                 transport="stdio",
                 command=python,
                 args=[script_path],
-                env={"PYTHONPATH": base_dir},
+                env=builtin_python_env(base_dir),
             )
             if ok:
                 logger.info(f"Built-in MCP server registered: {name}")

src/caldav_sync.py

@@ -280,214 +280,216 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i
     result = {"calendars": 0, "events": 0, "deleted": 0, "errors": []}
 
     client = _build_dav_client(url, username, password)
-
-    # Discovery: try principal → calendars first; if the server doesn't
-    # support discovery (or the URL points directly at a calendar), fall
-    # back to treating the URL as a single calendar.
-    calendars = []
     try:
-        principal = client.principal()
+        # Discovery: try principal → calendars first; if the server doesn't
-        calendars = principal.calendars()
+        # support discovery (or the URL points directly at a calendar), fall
-    except (AuthorizationError, NotFoundError) as e:
+        # back to treating the URL as a single calendar.
-        result["errors"].append(f"Discovery failed: {e}")
+        calendars = []
-        return result
-    except Exception as e:
-        logger.info(f"CalDAV principal discovery failed, trying URL as calendar: {e}")
         try:
-            calendars = [_open_url_as_calendar(client, url)]
+            principal = client.principal()
-        except Exception as e2:
+            calendars = principal.calendars()
-            result["errors"].append(f"Could not open URL as calendar: {e2}")
+        except (AuthorizationError, NotFoundError) as e:
-            return result
+            result["errors"].append(f"Discovery failed: {e}")
-
+            return result          # outer finally will call client.close()
-    if not calendars:
-        try:
-            calendars = [_open_url_as_calendar(client, url)]
         except Exception as e:
-            result["errors"].append(f"No calendars and URL fallback failed: {e}")
+            logger.info(f"CalDAV principal discovery failed, trying URL as calendar: {e}")
-            return result
-
-    start = datetime.utcnow() - timedelta(days=_LOOKBACK_DAYS)
-    end = datetime.utcnow() + timedelta(days=_LOOKAHEAD_DAYS)
-
-    db = SessionLocal()
-    try:
-        for remote_cal in calendars:
             try:
-                remote_url = str(remote_cal.url)
+                calendars = [_open_url_as_calendar(client, url)]
-                cal_id = _stable_cal_id(remote_url, owner=owner, account_id=account_id)
+            except Exception as e2:
-                display_name = (remote_cal.name or "").strip() or "CalDAV"
+                result["errors"].append(f"Could not open URL as calendar: {e2}")
+                return result      # outer finally will call client.close()
 
-                local_cal = db.query(CalendarCal).filter(
+        if not calendars:
-                    CalendarCal.id == cal_id,
+            try:
-                    CalendarCal.owner == owner,
+                calendars = [_open_url_as_calendar(client, url)]
-                ).first()
+            except Exception as e:
-                if not local_cal:
+                result["errors"].append(f"No calendars and URL fallback failed: {e}")
-                    local_cal = CalendarCal(
+                return result      # outer finally will call client.close()
-                        id=cal_id,
-                        owner=owner,
-                        name=display_name,
-                        color="#5b8abf",
-                        source="caldav",
-                        account_id=account_id or None,
-                        caldav_base_url=remote_url,
-                    )
-                    db.add(local_cal)
-                    db.commit()
-                else:
-                    # Refresh display name and stamp CalDAV metadata if missing.
-                    changed = False
-                    if local_cal.name != display_name:
-                        local_cal.name = display_name
-                        changed = True
-                    if account_id and not local_cal.account_id:
-                        local_cal.account_id = account_id
-                        changed = True
-                    if local_cal.caldav_base_url != remote_url:
-                        local_cal.caldav_base_url = remote_url
-                        changed = True
-                    if changed:
-                        db.commit()
-                result["calendars"] += 1
 
-                # Fetch events in window. `date_search` returns CalendarObject
+        start = datetime.utcnow() - timedelta(days=_LOOKBACK_DAYS)
-                # resources; each may contain one VEVENT (most servers) or
+        end = datetime.utcnow() + timedelta(days=_LOOKAHEAD_DAYS)
-                # several (rare).
-                from icalendar import Calendar as iCal
 
-                seen_uids = set()
+        db = SessionLocal()        # if this raises, outer finally still calls client.close()
-                # Track events added to the session but not yet committed so
+        try:
-                # duplicate UIDs within the same batch are updated, not re-inserted
+            for remote_cal in calendars:
-                # (which would violate the UNIQUE constraint on commit).
-                pending: dict = {}
-                parse_failed = False
                 try:
-                    objs = remote_cal.date_search(start=start, end=end, expand=False)
+                    remote_url = str(remote_cal.url)
-                except Exception as e:
+                    cal_id = _stable_cal_id(remote_url, owner=owner, account_id=account_id)
-                    result["errors"].append(f"{display_name}: date_search failed ({e})")
+                    display_name = (remote_cal.name or "").strip() or "CalDAV"
-                    continue
 
-                for obj in objs:
+                    local_cal = db.query(CalendarCal).filter(
+                        CalendarCal.id == cal_id,
+                        CalendarCal.owner == owner,
+                    ).first()
+                    if not local_cal:
+                        local_cal = CalendarCal(
+                            id=cal_id,
+                            owner=owner,
+                            name=display_name,
+                            color="#5b8abf",
+                            source="caldav",
+                            account_id=account_id or None,
+                            caldav_base_url=remote_url,
+                        )
+                        db.add(local_cal)
+                        db.commit()
+                    else:
+                        # Refresh display name and stamp CalDAV metadata if missing.
+                        changed = False
+                        if local_cal.name != display_name:
+                            local_cal.name = display_name
+                            changed = True
+                        if account_id and not local_cal.account_id:
+                            local_cal.account_id = account_id
+                            changed = True
+                        if local_cal.caldav_base_url != remote_url:
+                            local_cal.caldav_base_url = remote_url
+                            changed = True
+                        if changed:
+                            db.commit()
+                    result["calendars"] += 1
+
+                    # Fetch events in window. `date_search` returns CalendarObject
+                    # resources; each may contain one VEVENT (most servers) or
+                    # several (rare).
+                    from icalendar import Calendar as iCal
+
+                    seen_uids = set()
+                    # Track events added to the session but not yet committed so
+                    # duplicate UIDs within the same batch are updated, not re-inserted
+                    # (which would violates the UNIQUE constraint on commit).
+                    pending: dict = {}
+                    parse_failed = False
                     try:
-                        ical = iCal.from_ical(obj.data)
+                        objs = remote_cal.date_search(start=start, end=end, expand=False)
                     except Exception as e:
-                        result["errors"].append(f"{display_name}: parse failed ({e})")
+                        result["errors"].append(f"{display_name}: date_search failed ({e})")
-                        parse_failed = True
                         continue
 
-                    for comp in ical.walk():
+                    for obj in objs:
-                        if comp.name != "VEVENT":
+                        try:
+                            ical = iCal.from_ical(obj.data)
+                        except Exception as e:
+                            result["errors"].append(f"{display_name}: parse failed ({e})")
+                            parse_failed = True
                             continue
-                        uid_val = str(comp.get("uid", "")) or str(uuid.uuid4())
-                        seen_uids.add(uid_val)
 
-                        dtstart_p = comp.get("dtstart")
+                        for comp in ical.walk():
-                        if not dtstart_p:
+                            if comp.name != "VEVENT":
-                            continue
-                        start_dt, all_day = _to_utc_naive(dtstart_p.dt)
-
-                        dtend_p = comp.get("dtend")
-                        if dtend_p:
-                            end_dt, _ = _to_utc_naive(dtend_p.dt)
-                        elif all_day:
-                            end_dt = start_dt + timedelta(days=1)
-                        else:
-                            end_dt = start_dt + timedelta(hours=1)
-                        # A synced event with DTEND <= DTSTART (e.g. a single-day
-                        # all-day event whose source wrote DTEND equal to DTSTART)
-                        # would be stored zero-duration and silently dropped by the
-                        # list_events overlap filter. Clamp to a positive span.
-                        end_dt = _ensure_positive_duration(start_dt, end_dt, all_day)
-
-                        # is_utc reflects whether the source carried a TZ
-                        # we converted from. All-day = no TZ semantics.
-                        row_is_utc = (
-                            not all_day
-                            and isinstance(dtstart_p.dt, datetime)
-                            and dtstart_p.dt.tzinfo is not None
-                        )
-
-                        summary = str(comp.get("summary", ""))
-                        description = str(comp.get("description", ""))
-                        location = str(comp.get("location", ""))
-                        rrule = (
-                            comp.get("rrule").to_ical().decode()
-                            if comp.get("rrule")
-                            else ""
-                        )
-
-                        existing = _find_existing_event(db, pending, uid_val, local_cal.id)
-                        if existing:
-                            if existing.caldav_sync_pending in {"create", "update"}:
-                                result["events"] += 1
                                 continue
-                            existing.calendar_id = local_cal.id
+                            uid_val = str(comp.get("uid", "")) or str(uuid.uuid4())
-                            existing.summary = summary
+                            seen_uids.add(uid_val)
-                            existing.description = description
+
-                            existing.location = location
+                            dtstart_p = comp.get("dtstart")
-                            existing.dtstart = start_dt
+                            if not dtstart_p:
-                            existing.dtend = end_dt
+                                continue
-                            existing.all_day = all_day
+                            start_dt, all_day = _to_utc_naive(dtstart_p.dt)
-                            existing.is_utc = row_is_utc
+
-                            existing.rrule = rrule
+                            dtend_p = comp.get("dtend")
-                            existing.origin = "caldav"
+                            if dtend_p:
-                            existing.remote_href = str(getattr(obj, "url", "") or "") or None
+                                end_dt, _ = _to_utc_naive(dtend_p.dt)
-                            existing.remote_etag = _event_etag(obj) or None
+                            elif all_day:
-                            existing.caldav_sync_pending = None
+                                end_dt = start_dt + timedelta(days=1)
-                        else:
+                            else:
-                            new_ev = CalendarEvent(
+                                end_dt = start_dt + timedelta(hours=1)
-                                uid=uid_val,
+                            # A synced event with DTEND <= DTSTART (e.g. a single-day
-                                calendar_id=local_cal.id,
+                            # all-day event whose source wrote DTEND equal to DTSTART)
-                                summary=summary,
+                            # would be stored zero-duration and silently dropped by the
-                                description=description,
+                            # list_events overlap filter. Clamp to a positive span.
-                                location=location,
+                            end_dt = _ensure_positive_duration(start_dt, end_dt, all_day)
-                                dtstart=start_dt,
+
-                                dtend=end_dt,
+                            # is_utc reflects whether the source carried a TZ
-                                all_day=all_day,
+                            # we converted from. All-day = no TZ semantics.
-                                is_utc=row_is_utc,
+                            row_is_utc = (
-                                rrule=rrule,
+                                not all_day
-                                origin="caldav",
+                                and isinstance(dtstart_p.dt, datetime)
-                                remote_href=str(getattr(obj, "url", "") or "") or None,
+                                and dtstart_p.dt.tzinfo is not None
-                                remote_etag=_event_etag(obj) or None,
                             )
-                            db.add(new_ev)
-                            pending[uid_val] = new_ev
-                        result["events"] += 1
-                db.commit()
 
-                # Prune locally-cached CalDAV events that vanished
+                            summary = str(comp.get("summary", ""))
-                # upstream (only within our sync window — events outside
+                            description = str(comp.get("description", ""))
-                # the window aren't in `objs`, so we'd false-delete them).
+                            location = str(comp.get("location", ""))
-                # Only rows we previously pulled from the server (origin=="caldav")
+                            rrule = (
-                # are prunable; locally-created events (agent / email triage / a
+                                comp.get("rrule").to_ical().decode()
-                # UI event whose write-back failed) carry origin NULL and must
+                                if comp.get("rrule")
-                # never be deleted just because the server didn't return them.
+                                else ""
-                # Skip the prune on any parse failure: seen_uids is then an
+                            )
-                # incomplete view of the server, so pruning against it would
+
-                # delete events that still exist upstream but could not be read
+                            existing = _find_existing_event(db, pending, uid_val, local_cal.id)
-                # (the empty-seen_uids case wipes the whole window; a partial
+                            if existing:
-                # failure deletes just the unreadable rows).
+                                if existing.caldav_sync_pending in {"create", "update"}:
-                if _should_prune_window(seen_uids, parse_failed):
+                                    result["events"] += 1
-                    stale = db.query(CalendarEvent).filter(
+                                    continue
-                        CalendarEvent.calendar_id == local_cal.id,
+                                existing.calendar_id = local_cal.id
-                        CalendarEvent.origin == "caldav",
+                                existing.summary = summary
-                        CalendarEvent.dtstart >= start,
+                                existing.description = description
-                        CalendarEvent.dtstart <= end,
+                                existing.location = location
-                        CalendarEvent.remote_href.isnot(None),
+                                existing.dtstart = start_dt
-                        CalendarEvent.caldav_sync_pending.is_(None),
+                                existing.dtend = end_dt
-                        ~CalendarEvent.uid.in_(seen_uids) if seen_uids else CalendarEvent.uid.isnot(None),
+                                existing.all_day = all_day
-                    ).all()
+                                existing.is_utc = row_is_utc
-                    for ev in stale:
+                                existing.rrule = rrule
-                        db.delete(ev)
+                                existing.origin = "caldav"
-                    result["deleted"] += len(stale)
+                                existing.remote_href = str(getattr(obj, "url", "") or "") or None
+                                existing.remote_etag = _event_etag(obj) or None
+                                existing.caldav_sync_pending = None
+                            else:
+                                new_ev = CalendarEvent(
+                                    uid=uid_val,
+                                    calendar_id=local_cal.id,
+                                    summary=summary,
+                                    description=description,
+                                    location=location,
+                                    dtstart=start_dt,
+                                    dtend=end_dt,
+                                    all_day=all_day,
+                                    is_utc=row_is_utc,
+                                    rrule=rrule,
+                                    origin="caldav",
+                                    remote_href=str(getattr(obj, "url", "") or "") or None,
+                                    remote_etag=_event_etag(obj) or None,
+                                )
+                                db.add(new_ev)
+                                pending[uid_val] = new_ev
+                            result["events"] += 1
                     db.commit()
-            except Exception as e:
-                logger.exception("CalDAV sync failed for one calendar")
-                result["errors"].append(str(e)[:200])
-                db.rollback()
-    finally:
-        db.close()
 
-    return result
+                    # Prune locally-cached CalDAV events that vanished
+                    # upstream (only within our sync window — events outside
+                    # the window aren't in `objs`, so we'd false-delete them).
+                    # Only rows we previously pulled from the server (origin=="caldav")
+                    # are prunable; locally-created events (agent / email triage / a
+                    # UI event whose write-back failed) carry origin NULL and must
+                    # never be deleted just because the server didn't return them.
+                    # Skip the prune on any parse failure: seen_uids is then an
+                    # incomplete view of the server, so pruning against it would
+                    # delete events that still exist upstream but could not be read
+                    # (the empty-seen_uids case wipes the whole window; a partial
+                    # failure deletes just the unreadable rows).
+                    if _should_prune_window(seen_uids, parse_failed):
+                        stale = db.query(CalendarEvent).filter(
+                            CalendarEvent.calendar_id == local_cal.id,
+                            CalendarEvent.origin == "caldav",
+                            CalendarEvent.dtstart >= start,
+                            CalendarEvent.dtstart <= end,
+                            CalendarEvent.remote_href.isnot(None),
+                            CalendarEvent.caldav_sync_pending.is_(None),
+                            ~CalendarEvent.uid.in_(seen_uids) if seen_uids else CalendarEvent.uid.isnot(None),
+                        ).all()
+                        for ev in stale:
+                            db.delete(ev)
+                        result["deleted"] += len(stale)
+                        db.commit()
+                except Exception as e:
+                    logger.exception("CalDAV sync failed for one calendar")
+                    result["errors"].append(str(e)[:200])
+                    db.rollback()
+        finally:
+            db.close()             # NOT client.close() here anymore
+
+        return result
+    finally:
+        client.close()             # always called
 
 
 def _event_payload(ev) -> dict:

src/caldav_writeback.py

@@ -192,11 +192,14 @@ def _writeback_blocking(local_cal_id, ev, delete, url, username, password,
     # Redirects disabled here too: the write-back path opens its own DAVClient,
     # so it needs the same SSRF-via-redirect protection as the pull path.
     client = _build_dav_client(url, username, password)
-    calendars = _discover_calendars(client)
+    try:
-    if not calendars:
+        calendars = _discover_calendars(client)
-        return {"ok": False, "error": "no remote calendars discovered"}
+        if not calendars:
-    return push_event(calendars, local_cal_id, ev, delete=delete,
+            return {"ok": False, "error": "no remote calendars discovered"}
-                      owner=owner, account_id=account_id)
+        return push_event(calendars, local_cal_id, ev, delete=delete,
+                          owner=owner, account_id=account_id)
+    finally:
+        client.close()
 
 
 def _persist_writeback_result(owner: str, calendar_id: str, uid: str, result: dict, *, delete: bool) -> None:

src/chat_handler.py

@@ -191,6 +191,8 @@ class ChatHandler:
                         "name": fi.get("name") or fi.get("original_name") or fi["id"],
                         "mime": fi.get("mime", ""),
                         "size": fi.get("size", 0),
+                        "checksum_sha256": fi.get("checksum_sha256") or fi.get("hash"),
+                        "created_at": fi.get("created_at") or fi.get("uploaded_at"),
                         "width": fi.get("width"),
                         "height": fi.get("height"),
                     })

src/config.py

@@ -27,7 +27,6 @@ class DataConfig(BaseSettings):
     uploads_dir: Path = Field(default=Path(_DATA_DIR_CONST) / "uploads", description="Directory for uploaded files")
     sessions_file: Path = Field(default=Path(_DATA_DIR_CONST) / "sessions.json", description="Sessions storage file")
     memory_file: Path = Field(default=Path(_DATA_DIR_CONST) / "memory.json", description="Memory storage file")
-    memory_doc: Path = Field(default=Path(_DATA_DIR_CONST) / "memory_doc.md", description="Memory document file")
     personal_dir: Path = Field(default=Path(_DATA_DIR_CONST) / "personal_docs", description="Personal documents directory")
     runbook_dir: Path = Field(default=Path(_DATA_DIR_CONST) / "personal_docs" / "runbook", description="Runbook directory")
     
@@ -162,7 +161,6 @@ class AppConfig(BaseSettings):
             "uploads_dir": data_dir / "uploads",
             "sessions_file": data_dir / "sessions.json",
             "memory_file": data_dir / "memory.json",
-            "memory_doc": data_dir / "memory_doc.md",
             "personal_dir": data_dir / "personal_docs",
             "runbook_dir": data_dir / "personal_docs" / "runbook",
             "max_upload_size": max_upload_size,

src/constants.py

@@ -17,7 +17,6 @@ DATA_DIR = os.getenv("ODYSSEUS_DATA_DIR", get_default_data_dir())
 # re-deriving paths from __file__ or a relative "data" literal.
 SESSIONS_FILE = os.path.join(DATA_DIR, "sessions.json")
 MEMORY_FILE = os.path.join(DATA_DIR, "memory.json")
-MEMORY_DOC = os.path.join(DATA_DIR, "memory_doc.md")
 PERSONAL_DIR = os.path.join(DATA_DIR, "personal_docs")
 RUNBOOK_DIR = os.path.join(PERSONAL_DIR, "runbook")
 UPLOAD_DIR = os.path.join(DATA_DIR, "uploads")

src/copilot.py

@@ -230,7 +230,10 @@ def request_flags(messages) -> tuple:
     """
     msgs = messages or []
     last = msgs[-1] if msgs else None
-    agent = bool(last) and last.get("role") != "user"
+    # A message element can be a non-dict (clients send `"messages": ["hi"]`);
+    # the vision loop below already guards each element with isinstance, so do
+    # the same here rather than call .get() on a bare string.
+    agent = isinstance(last, dict) and last.get("role") != "user"
     vision = False
     for m in msgs:
         content = m.get("content") if isinstance(m, dict) else None

src/document_processor.py

@@ -440,7 +440,10 @@ def build_user_content(
             try:
                 with open(path, "rb") as image_file:
                     encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
-                image_format = ext[1:]
+                # Extensionless uploads (e.g. a pasted screenshot) have no ext,
+                # so fall back to the resolved MIME subtype rather than emitting
+                # an invalid "data:image/;base64," with an empty subtype.
+                image_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("image/") else "png")
                 content.append({
                     "type": "image_url",
                     "image_url": {"url": f"data:image/{image_format};base64,{encoded_string}"},
@@ -456,7 +459,7 @@ def build_user_content(
             try:
                 with open(path, "rb") as audio_file:
                     encoded_string = base64.b64encode(audio_file.read()).decode("utf-8")
-                audio_format = ext[1:]
+                audio_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("audio/") else "mpeg")
                 content.append({
                     "type": "audio",
                     "audio": {"url": f"data:audio/{audio_format};base64,{encoded_string}"},

src/endpoint_resolver.py

@@ -47,6 +47,33 @@ def _endpoint_cached_models(ep) -> list:
     return models if isinstance(models, list) else []
 
 
+def _endpoint_pinned_models(ep) -> list:
+    raw = getattr(ep, "pinned_models", None)
+    if not raw:
+        return []
+    try:
+        models = json.loads(raw) if isinstance(raw, str) else raw
+    except Exception:
+        return []
+    return models if isinstance(models, list) else []
+
+
+def _is_mlx_deepseek_v4_repo_id(model_id: str) -> bool:
+    return "mlx-community/deepseek-v4" in str(model_id or "").lower()
+
+
+def _is_mlx_deepseek_v4_shim_id(model_id: str) -> bool:
+    return "/.cache/odysseus/mlx-shims/deepseek-v4" in str(model_id or "").lower()
+
+
+def _filter_mlx_deepseek_v4_repo_when_shimmed(model_ids) -> list:
+    ids = list(model_ids or [])
+    has_shim = any(_is_mlx_deepseek_v4_shim_id(m) for m in ids)
+    if not has_shim:
+        return ids
+    return [m for m in ids if not _is_mlx_deepseek_v4_repo_id(m)]
+
+
 def _endpoint_hidden_models(ep) -> set:
     """Model ids the admin disabled on this endpoint (the UI's hidden list)."""
     raw = getattr(ep, "hidden_models", None)
@@ -67,7 +94,15 @@ def _endpoint_enabled_models(ep) -> list:
     raw first one resolves to a model that 400s ("requires terms acceptance").
     """
     hidden = _endpoint_hidden_models(ep)
-    return [m for m in _endpoint_cached_models(ep) if m not in hidden]
+    merged = []
+    seen = set()
+    for m in [*_endpoint_cached_models(ep), *_endpoint_pinned_models(ep)]:
+        if not isinstance(m, str) or not m or m in seen:
+            continue
+        seen.add(m)
+        merged.append(m)
+    merged = _filter_mlx_deepseek_v4_repo_when_shimmed(merged)
+    return [m for m in merged if m not in hidden]
 
 
 def resolve_endpoint_runtime(ep, owner: Optional[str] = None) -> Tuple[str, Optional[str]]:

src/integrations.py

@@ -216,7 +216,14 @@ def _normalize_integration_base_url(base_url: Any) -> str:
 
 
 def _join_integration_url(base_url: str, path: str) -> str:
-    return urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
+    base = base_url.rstrip("/")
+    rel = path.lstrip("/")
+    if not rel:
+        # A bare "/" must resolve to the base URL itself, not base + "/".
+        # POST-to-base integrations (e.g. Discord webhooks) 404 on the
+        # trailing-slash variant of their URL.
+        return base
+    return urljoin(base + "/", rel)
 
 
 def load_integrations() -> List[Dict[str, Any]]:
@@ -394,6 +401,22 @@ async def execute_api_call(
         return {"error": "Path must not contain a fragment", "exit_code": 1}
 
     url = _join_integration_url(base_url, path)
+
+    # SSRF guard — same check used by the gallery endpoint, embeddings,
+    # CardDAV, and the reminder webhook sender. Link-local / metadata
+    # addresses (169.254.x.x — the cloud credential-exfil vector) are always
+    # rejected; INTEGRATION_API_BLOCK_PRIVATE_IPS=true also blocks RFC-1918 /
+    # loopback for locked-down deployments. Private stays allowed by default
+    # because LAN integrations (Home Assistant, Miniflux, ntfy) are the
+    # primary use case.
+    from src.url_safety import check_outbound_url
+    block_private = os.getenv(
+        "INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
+    ).lower() == "true"
+    ok, reason = check_outbound_url(url, block_private=block_private)
+    if not ok:
+        return {"error": f"URL rejected: {reason}", "exit_code": 1}
+
     method = method.upper()
 
     # Build headers

src/interactive_gate.py

@@ -17,6 +17,7 @@ _ACTIVE_REQUESTS = 0
 _LAST_ACTIVITY = 0.0
 _LAST_BROWSER_ACTIVITY = 0.0
 _COND: asyncio.Condition | None = None
+_COND_LOOP: asyncio.AbstractEventLoop | None = None
 
 
 def _enabled() -> bool:
@@ -47,14 +48,20 @@ def _browser_active_seconds() -> float:
 
 
 def _condition() -> asyncio.Condition:
-    global _COND
+    global _COND, _COND_LOOP
-    if _COND is None:
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+    if _COND is None or _COND_LOOP is not loop:
         _COND = asyncio.Condition()
+        _COND_LOOP = loop
     return _COND
 
 
 _PASSIVE_EXACT_PATHS = {
     "/api/activity/heartbeat",
+    "/api/client-perf",
     "/api/tasks/notifications",
     "/api/research/active",
     "/api/email/urgency-state",
@@ -100,15 +107,18 @@ def _has_recent_browser_activity(now: float | None = None) -> bool:
 def has_foreground_activity(now: float | None = None) -> bool:
     """Return True when foreground browser/model work should stop background jobs.
 
-    This is intentionally narrower than `wait_for_interactive_quiet`: active
+    Passive polling endpoints are excluded by should_track_interactive_request,
-    request tracking is good for delaying task startup, but a running task
+    so active/recent request tracking is safe to use here. This matters during
-    should not cancel itself just because the UI polls a passive endpoint.
+    initial page load: the heartbeat may not have landed yet, but the user is
-    Browser heartbeats and active chat streams are the durable "user is here"
+    already waiting on real UI requests.
-    signals.
     """
     if not _enabled():
         return False
     t = now if now is not None else time.monotonic()
+    if _ACTIVE_REQUESTS > 0:
+        return True
+    if _LAST_ACTIVITY > 0 and (t - _LAST_ACTIVITY) < _quiet_seconds():
+        return True
     return _has_recent_browser_activity(t) or _has_active_chat_stream()
 
 

src/llm_core.py

@@ -8,13 +8,88 @@ import hashlib
 import threading
 import re
 import os
+from contextlib import asynccontextmanager
 from fastapi import HTTPException
 from typing import Optional, Dict, List, Tuple
-from src.model_context import get_context_length, DEFAULT_CONTEXT
+from src.model_context import get_context_length, DEFAULT_CONTEXT, is_local_endpoint
 from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
+_LOCAL_MODEL_LOCK = asyncio.Lock()
+_LOCAL_MODEL_WAITING_FOREGROUND = 0
+_LOCAL_MODEL_CURRENT: Dict[str, object] = {}
+
+
+def _local_model_gate_enabled() -> bool:
+    return os.getenv("ODYSSEUS_LOCAL_MODEL_GATE", "true").lower() not in {"0", "false", "no", "off"}
+
+
+def _gate_workload(workload: Optional[str]) -> str:
+    return "background" if str(workload or "").lower() == "background" else "foreground"
+
+
+@asynccontextmanager
+async def _local_model_slot(target_url: str, model: str, workload: Optional[str] = None):
+    """Serialize local model traffic, with foreground chat taking priority.
+
+    Most local servers expose one GPU/CPU generation pipe even when their HTTP
+    API accepts multiple requests. Letting scheduled email/tasks and foreground
+    chat hit that pipe together creates the user-visible "streams crossed" and
+    "prompt waited behind a task" failure mode. Cloud providers are left alone.
+    """
+    if not _local_model_gate_enabled() or not is_local_endpoint(target_url):
+        yield
+        return
+
+    global _LOCAL_MODEL_WAITING_FOREGROUND
+    kind = _gate_workload(workload)
+    current_task = asyncio.current_task()
+    if kind == "foreground":
+        _LOCAL_MODEL_WAITING_FOREGROUND += 1
+        current = dict(_LOCAL_MODEL_CURRENT)
+        if current.get("workload") == "background":
+            task = current.get("task")
+            if isinstance(task, asyncio.Task) and not task.done():
+                logger.info(
+                    "[model-gate] cancelling background local model call for foreground request model=%s",
+                    model,
+                )
+                task.cancel()
+    else:
+        # Background work should not jump in while the browser/chat is active
+        # or while a foreground request is waiting to acquire the local model.
+        try:
+            from src.interactive_gate import has_foreground_activity
+        except Exception:
+            has_foreground_activity = lambda: False  # type: ignore
+        while _LOCAL_MODEL_WAITING_FOREGROUND > 0 or has_foreground_activity():
+            await asyncio.sleep(0.25)
+
+    acquired = False
+    try:
+        await _LOCAL_MODEL_LOCK.acquire()
+        acquired = True
+        if kind == "foreground":
+            _LOCAL_MODEL_WAITING_FOREGROUND = max(0, _LOCAL_MODEL_WAITING_FOREGROUND - 1)
+        _LOCAL_MODEL_CURRENT.clear()
+        _LOCAL_MODEL_CURRENT.update({
+            "task": current_task,
+            "workload": kind,
+            "url": target_url,
+            "model": model,
+            "started": time.time(),
+        })
+        yield
+    finally:
+        if kind == "foreground":
+            _LOCAL_MODEL_WAITING_FOREGROUND = max(0, _LOCAL_MODEL_WAITING_FOREGROUND - 1)
+        if acquired and _LOCAL_MODEL_LOCK.locked():
+            owner = _LOCAL_MODEL_CURRENT.get("task")
+            if owner is current_task:
+                _LOCAL_MODEL_CURRENT.clear()
+            _LOCAL_MODEL_LOCK.release()
+
 class LLMConfig:
     """Configuration constants for LLM operations."""
     DEFAULT_TIMEOUT = 30
@@ -199,6 +274,75 @@ def _stream_delta_event(text: str, *, thinking: bool = False) -> str:
         payload["thinking"] = True
     return f"data: {json.dumps(payload)}\n\n"
 
+
+_DEGENERATE_WORD_RE = re.compile(r"[A-Za-z0-9_\u0370-\u03ff\u0400-\u04ff]+")
+
+
+class _DegenerateStreamGuard:
+    """Detect local-model token collapse before it floods the UI.
+
+    Some self-hosted models fail by repeating one token forever ("Var Var Var",
+    "Summer Summer ..."). This is not a useful response and can burn context,
+    browser memory, and GPU time. Keep the guard conservative: only fire on long
+    same-token runs or a very dominant repeated token in the recent window.
+    """
+
+    def __init__(self, model: str):
+        self.model = model or "model"
+        self.last_token = ""
+        self.same_run = 0
+        self.recent_tokens: List[str] = []
+        self.total_chars = 0
+
+    def check(self, text: str) -> Optional[str]:
+        if not text:
+            return None
+        self.total_chars += len(text)
+        tokens = [t.lower() for t in _DEGENERATE_WORD_RE.findall(text) if len(t) >= 2]
+        if not tokens:
+            return None
+        for token in tokens:
+            if token == self.last_token:
+                self.same_run += 1
+            else:
+                self.last_token = token
+                self.same_run = 1
+            self.recent_tokens.append(token)
+        if len(self.recent_tokens) > 96:
+            self.recent_tokens = self.recent_tokens[-96:]
+
+        reason = None
+        if self.same_run >= 28 and self.total_chars >= 100:
+            reason = f"repeated '{self.last_token}' {self.same_run} times"
+        elif len(self.recent_tokens) >= 72:
+            top = max(set(self.recent_tokens), key=self.recent_tokens.count)
+            count = self.recent_tokens.count(top)
+            if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78:
+                reason = f"repeated '{top}' {count}/{len(self.recent_tokens)} recent tokens"
+        if not reason and len(self.recent_tokens) >= 80:
+            # Phrase loops are common on some local quantized MLX/MoE models:
+            # "Also be a software developer mode?" repeated forever will not
+            # trip the single-token guard above, but it is still a wedged
+            # generation. Require many repeats of the same 4-gram so normal
+            # prose/list formatting is not interrupted.
+            grams = [tuple(self.recent_tokens[i:i + 4]) for i in range(0, len(self.recent_tokens) - 3)]
+            if grams:
+                top_gram = max(set(grams), key=grams.count)
+                gram_count = grams.count(top_gram)
+                if gram_count >= 10:
+                    reason = f"repeated phrase '{' '.join(top_gram)}' {gram_count} times"
+
+        if not reason:
+            return None
+
+        logger.warning("[degenerate-stream] aborting model=%s reason=%s", self.model, reason)
+        message = (
+            f"Stopped generation: {self.model} started repeating tokens "
+            f"({reason}). Try a different model or lower temperature."
+        )
+        return f'event: error\ndata: {json.dumps({"status": 502, "text": message, "error": message})}\n\n'
+
+
 def _model_activity_key(url: str, model: str) -> str:
     return f"{(url or '').strip()}|{(model or '').strip()}"
 
@@ -622,6 +766,36 @@ def apply_kimi_code_headers(headers: Optional[Dict], url: str) -> Dict[str, str]
     return h
 
 
+async def apply_kimi_code_headers_async(client, headers: Optional[Dict], url: str) -> Dict[str, str]:
+    """Pick a Kimi Code User-Agent without blocking the event loop."""
+    h = dict(headers or {})
+    if not _is_kimi_code_url(url):
+        return h
+    base_key = _kimi_code_base_key(url)
+    cached = _kimi_code_ua_cache.get(base_key)
+    if cached:
+        h["User-Agent"] = cached
+        return h
+    models_url = base_key.rstrip("/") + "/models"
+    for ua in KIMI_CODE_USER_AGENTS:
+        trial = dict(h)
+        trial["User-Agent"] = ua
+        try:
+            r = await client.get(models_url, headers=trial, timeout=8)
+        except Exception:
+            continue
+        if _is_kimi_code_access_denied(r.status_code, r.content):
+            logger.debug("Kimi Code rejected User-Agent %s (403), trying next", ua)
+            continue
+        if r.status_code < 400:
+            _remember_kimi_code_user_agent(url, ua)
+            h["User-Agent"] = ua
+            return h
+        break
+    h.setdefault("User-Agent", KIMI_CODE_USER_AGENT)
+    return h
+
+
 def httpx_get_kimi_aware(url: str, headers: Optional[Dict], **kwargs):
     h = apply_kimi_code_headers(headers, url)
     if not _is_kimi_code_url(url):
@@ -655,7 +829,7 @@ def httpx_post_kimi_aware(url: str, headers: Optional[Dict], **kwargs):
 
 
 async def httpx_post_kimi_aware_async(client, url: str, headers: Optional[Dict], **kwargs):
-    h = apply_kimi_code_headers(headers, url)
+    h = await apply_kimi_code_headers_async(client, headers, url)
     if not _is_kimi_code_url(url):
         return await client.post(url, headers=h, **kwargs)
     last = None
@@ -755,6 +929,52 @@ def _apply_local_cache_affinity(payload: Dict, url: str, session_id: Optional[st
     payload.setdefault("cache_prompt", True)
 
 
+def _is_local_minimax_mlx_request(url: str, model: str) -> bool:
+    """Local MLX MiniMax-family endpoints need conservative sampling defaults.
+
+    The OpenAI-compatible MLX server accepts repetition/frequency penalties.
+    Some large quantized MiniMax/MoE ports otherwise fall into visible reasoning
+    loops ("Also be...", "No.", etc.) even for trivial prompts.
+    """
+    if not model:
+        return False
+    m = model.lower()
+    if "minimax" not in m and "mini-max" not in m:
+        return False
+    try:
+        from src.model_context import is_local_endpoint
+        return is_local_endpoint(url)
+    except Exception:
+        return False
+
+
+def _apply_local_generation_stability(payload: Dict, url: str, model: str) -> None:
+    if not _is_local_minimax_mlx_request(url, model):
+        return
+    if "temperature" in payload:
+        try:
+            # MiniMax MLX quantized ports are very sensitive to chat/agent
+            # harness size. Character presets can ask for a warmer voice, but
+            # local MiniMax needs a final compatibility clamp or trivial
+            # prompts can fall into visible reasoning/repetition loops.
+            payload["temperature"] = min(float(payload.get("temperature") or 0.2), 0.2)
+        except (TypeError, ValueError):
+            payload["temperature"] = 0.2
+    payload.setdefault("top_p", 0.9)
+    payload.setdefault("top_k", 20)
+    payload.setdefault("repetition_penalty", 1.12)
+    payload.setdefault("repetition_context_size", 256)
+    payload.setdefault("frequency_penalty", 0.08)
+    payload.setdefault("frequency_context_size", 256)
+    payload.setdefault("presence_penalty", 0.02)
+    payload.setdefault("presence_context_size", 256)
+    payload.setdefault("stop", ["<|im_end|>", "<|endoftext|>", "</s>"])
+    # A max_tokens of 0 means "server default/unbounded" for many local
+    # endpoints. Keep simple chats from running forever when the model loops.
+    if not payload.get("max_tokens") and not payload.get("max_completion_tokens"):
+        payload["max_tokens"] = 2048
+
+
 def _provider_headers(provider: str, headers: Optional[Dict] = None) -> Dict[str, str]:
     h = {"Content-Type": "application/json"}
     if isinstance(headers, dict):
@@ -1602,6 +1822,7 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
         if max_tokens and max_tokens > 0:
             tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
             payload[tok_key] = max_tokens
+        _apply_local_generation_stability(payload, target_url, model)
         if provider == "mistral" and _supports_thinking(model):
             payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
     try:
@@ -1711,6 +1932,7 @@ async def llm_call_async(
     max_retries: int = LLMConfig.MAX_RETRIES,
     prompt_type: Optional[str] = None,
     session_id: Optional[str] = None,
+    workload: str = "foreground",
 ) -> str:
     """Asynchronous LLM call using httpx with connection pooling, timeout, retry logic, and performance logging."""
     provider = _detect_provider(url)
@@ -1748,6 +1970,7 @@ async def llm_call_async(
             max_tokens=max_tokens,
             headers=headers,
             timeout=timeout,
+            workload=workload,
         ):
             event_is_error = False
             for line in str(chunk).splitlines():
@@ -1813,6 +2036,7 @@ async def llm_call_async(
         if provider == "mistral" and _supports_thinking(model):
             payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
         _apply_local_cache_affinity(payload, url, session_id)
+        _apply_local_generation_stability(payload, target_url, model)
 
     if _is_host_dead(target_url):
         raise HTTPException(503, f"Upstream {_host_key(target_url)} marked unreachable (cooldown active)")
@@ -1823,9 +2047,10 @@ async def llm_call_async(
         attempt += 1
         start = time.time()
         try:
-            note_model_activity(target_url, model)
+            async with _local_model_slot(target_url, model, workload):
-            client = _get_http_client()
+                note_model_activity(target_url, model)
-            r = await httpx_post_kimi_aware_async(client, target_url, h, json=payload, timeout=call_timeout)
+                client = _get_http_client()
+                r = await httpx_post_kimi_aware_async(client, target_url, h, json=payload, timeout=call_timeout)
             duration = time.time() - start
             if not r.is_success:
                 friendly = _format_upstream_error(r.status_code, r.text, target_url)
@@ -1867,11 +2092,45 @@ async def llm_call_async(
                 raise HTTPException(502, f"POST {target_url} failed after {max_retries} attempts: {e}")
             await asyncio.sleep(LLMConfig.RETRY_DELAY)
 
+def _stream_target_url(url: str) -> str:
+    provider = _detect_provider(url)
+    if provider == "anthropic":
+        return _normalize_anthropic_url(url)
+    if provider == "ollama":
+        return _normalize_ollama_url(url)
+    if provider == "chatgpt-subscription":
+        return _normalize_chatgpt_subscription_url(url)
+    return _normalize_openai_chat_url(url)
+
+
 async def stream_llm(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
                      max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
                      timeout: int = LLMConfig.STREAM_TIMEOUT, prompt_type: Optional[str] = None,
                      tools: Optional[List[Dict]] = None, session_id: Optional[str] = None,
-                     tool_choice_none: bool = False):
+                     tool_choice_none: bool = False, workload: str = "foreground"):
+    target_url = _stream_target_url(url)
+    async with _local_model_slot(target_url, model, workload):
+        async for chunk in _stream_llm_inner(
+            url,
+            model,
+            messages,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            headers=headers,
+            timeout=timeout,
+            prompt_type=prompt_type,
+            tools=tools,
+            session_id=session_id,
+            tool_choice_none=tool_choice_none,
+        ):
+            yield chunk
+
+
+async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
+                            max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
+                            timeout: int = LLMConfig.STREAM_TIMEOUT, prompt_type: Optional[str] = None,
+                            tools: Optional[List[Dict]] = None, session_id: Optional[str] = None,
+                            tool_choice_none: bool = False):
     """Stream LLM responses with improved error handling.
 
     Yields SSE chunks:
@@ -1945,6 +2204,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
         if _is_ollama_openai_compat_url(url) and _supports_thinking(model):
             payload["think"] = False
         _apply_local_cache_affinity(payload, url, session_id)
+        _apply_local_generation_stability(payload, target_url, model)
         h = _provider_headers(provider, headers)
         if provider == "copilot":
             from src.copilot import apply_request_headers
@@ -1961,6 +2221,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
         yield f'event: error\ndata: {json.dumps({"error": f"Upstream {_host_key(target_url)} unreachable (cooldown active)", "status": 503})}\n\n'
         return
     note_model_activity(target_url, model)
+    degenerate_guard = _DegenerateStreamGuard(model)
 
     # ── ChatGPT Subscription / Codex Responses streaming ──
     if provider == "chatgpt-subscription":
@@ -1995,6 +2256,10 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
                     if evt == "response.output_text.delta":
                         delta = data.get("delta") or ""
                         if delta:
+                            _degenerate = degenerate_guard.check(delta)
+                            if _degenerate:
+                                yield _degenerate
+                                return
                             yield f'data: {json.dumps({"delta": delta})}\n\n'
                     elif evt == "response.completed":
                         usage = (data.get("response") or {}).get("usage") or data.get("usage") or {}
@@ -2231,9 +2496,9 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
             events.append(_stream_delta_event(part))
         return events
 
-    h = apply_kimi_code_headers(h, target_url)
     try:
         client = _get_http_client()
+        h = await apply_kimi_code_headers_async(client, h, target_url)
         async with client.stream('POST', target_url, json=payload, headers=h, timeout=stream_timeout) as r:
             _clear_host_dead(target_url)
             if r.status_code != 200:
@@ -2327,11 +2592,19 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
                                                 reasoning = (reasoning + thinking_part) if reasoning else thinking_part
                                             content = text_part
                                         if reasoning:
+                                            _degenerate = degenerate_guard.check(reasoning)
+                                            if _degenerate:
+                                                yield _degenerate
+                                                return
                                             yield _stream_delta_event(reasoning, thinking=True)
                                         if content:
                                             content = _strip_visible_chat_template_artifacts(content)
                                             if not content:
                                                 continue
+                                            _degenerate = degenerate_guard.check(content)
+                                            if _degenerate:
+                                                yield _degenerate
+                                                return
                                             content = re.sub(r"<mm:think(\s+[^>]*)?>", r"<think\1>", content, flags=re.IGNORECASE)
                                             content = re.sub(r"</mm:think>", "</think>", content, flags=re.IGNORECASE)
                                             stripped = content.lstrip()

src/mcp_manager.py

@@ -9,7 +9,9 @@ import json
 import logging
 import os
 import re
+import asyncio 
 from typing import Any, Dict, List, Optional, Set, Tuple
+from src.database import McpServer, SessionLocal
 
 from src.runtime_paths import get_app_root
 
@@ -192,57 +194,65 @@ class McpManager:
             )
 
             stack = AsyncExitStack()
+            registered = False
+
             try:
                 transport = await stack.enter_async_context(stdio_client(server_params))
                 read_stream, write_stream = transport
                 session = await stack.enter_async_context(ClientSession(read_stream, write_stream))
 
                 await session.initialize()
-
-                # Discover tools
                 tools_result = await session.list_tools()
-            except Exception:
-                await stack.aclose()
-                raise
-            tools = []
-            for tool in tools_result.tools:
-                tools.append({
-                    "name": tool.name,
-                    "description": tool.description or "",
-                    "input_schema": tool.inputSchema if hasattr(tool, 'inputSchema') else {},
-                    # MCP tool annotations (readOnlyHint / destructiveHint) drive
-                    # plan-mode read-only gating. Absent on many servers, so we
-                    # fall back to a name heuristic in mcp_tool_is_readonly().
-                    "annotations": getattr(tool, 'annotations', None),
-                })
 
-            self._sessions[server_id] = session
+                tools = []
-            self._stacks[server_id] = stack
+                for tool in tools_result.tools:
-            self._tools[server_id] = tools
+                    tools.append({
-            # Extract identity hints from env vars (e.g. email address, API name)
+                        "name": tool.name,
-            # so tool descriptions can distinguish between multiple instances of
+                        "description": tool.description or "",
-            # the same MCP server (e.g. two email accounts).
+                        "input_schema": tool.inputSchema if hasattr(tool, "inputSchema") else {},
-            identity_hints = []
+                        # MCP tool annotations (readOnlyHint / destructiveHint) drive
-            for k, v in (env or {}).items():
+                        # plan-mode read-only gating. Absent on many servers, so we
-                k_lower = k.lower()
+                        # fall back to a name heuristic in mcp_tool_is_readonly().
-                if any(x in k_lower for x in ['email_address', 'account', 'user', 'username']):
+                        "annotations": getattr(tool, "annotations", None),
-                    identity_hints.append(v)
+                    })
-            identity = ", ".join(identity_hints) if identity_hints else ""
 
-            self._connections[server_id] = {
+                # Extract identity hints from env vars (e.g. email address, API name)
-                "status": "connected",
+                # so tool descriptions can distinguish between multiple instances of
-                "name": name,
+                # the same MCP server (e.g. two email accounts).
-                "transport": "stdio",
+                identity_hints = []
-                "tool_count": len(tools),
+                for k, v in (env or {}).items():
-                "identity": identity,
+                    k_lower = k.lower()
-            }
+                    if any(x in k_lower for x in ["email_address", "account", "user", "username"]):
+                        identity_hints.append(v)
+                identity = ", ".join(identity_hints) if identity_hints else ""
+
+                self._sessions[server_id] = session
+                self._stacks[server_id] = stack
+                self._tools[server_id] = tools
+                self._connections[server_id] = {
+                    "status": "connected",
+                    "name": name,
+                    "transport": "stdio",
+                    "tool_count": len(tools),
+                    "identity": identity,
+                }
+
+                registered = True
+
+            finally:
+                if not registered:
+                    await stack.aclose()
 
             logger.info(f"MCP server connected: {name} ({server_id}) - {len(tools)} tools via stdio")
             return True
 
         except ImportError:
             logger.warning("MCP package not installed. Install with: pip install mcp")
-            self._connections[server_id] = {"status": "error", "error": "mcp package not installed", "name": name}
+            self._connections[server_id] = {
+                "status": "error",
+                "error": "mcp package not installed",
+                "name": name,
+            }
             return False
 
     async def _connect_sse(self, server_id: str, name: str, url: str) -> bool:
@@ -253,42 +263,46 @@ class McpManager:
             from contextlib import AsyncExitStack
 
             stack = AsyncExitStack()
+            registered = False
+
             try:
                 transport = await stack.enter_async_context(sse_client(url))
                 read_stream, write_stream = transport
                 session = await stack.enter_async_context(ClientSession(read_stream, write_stream))
 
                 await session.initialize()
-
-                # Discover tools
                 tools_result = await session.list_tools()
-            except Exception:
-                await stack.aclose()
-                raise
-            tools = []
-            for tool in tools_result.tools:
-                tools.append({
-                    "name": tool.name,
-                    "description": tool.description or "",
-                    "input_schema": tool.inputSchema if hasattr(tool, 'inputSchema') else {},
-                    # MCP tool annotations (readOnlyHint / destructiveHint) drive
-                    # plan-mode read-only gating. Absent on many servers, so we
-                    # fall back to a name heuristic in mcp_tool_is_readonly().
-                    "annotations": getattr(tool, 'annotations', None),
-                })
 
-            self._sessions[server_id] = session
+                tools = []
-            self._stacks[server_id] = stack
+                for tool in tools_result.tools:
-            self._tools[server_id] = tools
+                    tools.append({
-            self._connections[server_id] = {
+                        "name": tool.name,
-                "status": "connected",
+                        "description": tool.description or "",
-                "name": name,
+                        "input_schema": tool.inputSchema if hasattr(tool, 'inputSchema') else {},
-                "transport": "sse",
+                        # MCP tool annotations (readOnlyHint / destructiveHint) drive
-                "tool_count": len(tools),
+                        # plan-mode read-only gating. Absent on many servers, so we
-            }
+                        # fall back to a name heuristic in mcp_tool_is_readonly().
+                        "annotations": getattr(tool, 'annotations', None),
+                    })
 
-            logger.info(f"MCP server connected: {name} ({server_id}) - {len(tools)} tools via SSE")
+                self._sessions[server_id] = session
-            return True
+                self._stacks[server_id] = stack
+                self._tools[server_id] = tools
+                self._connections[server_id] = {
+                    "status": "connected",
+                    "name": name,
+                    "transport": "sse",
+                    "tool_count": len(tools),
+                }
+
+                registered = True
+
+                logger.info(f"MCP server connected: {name} ({server_id}) - {len(tools)} tools via SSE")
+                return True
+
+            finally:
+                if not registered:
+                    await stack.aclose()
 
         except ImportError:
             logger.warning("MCP package not installed. Install with: pip install mcp")
@@ -409,17 +423,29 @@ class McpManager:
         for sid in ids:
             await self.disconnect_server(sid)
 
-    async def connect_all_enabled(self):
-        """Connect to all enabled MCP servers from the database."""
-        from src.database import McpServer, SessionLocal
 
+    async def connect_all_enabled(self):
         db = SessionLocal()
         try:
             servers = db.query(McpServer).filter(McpServer.is_enabled == True).all()
-            for srv in servers:
+
-                args = json.loads(srv.args) if srv.args else []
+            tasks = [
-                env = json.loads(srv.env) if srv.env else {}
+                asyncio.create_task(self._connect_with_timeout(srv))
-                await self.connect_server(
+                for srv in servers
+            ]
+
+            await asyncio.gather(*tasks)
+        finally:
+            db.close()
+
+
+    async def _connect_with_timeout(self, srv):
+        args = json.loads(srv.args) if srv.args else []
+        env = json.loads(srv.env) if srv.env else {}
+
+        try:
+            await asyncio.wait_for(
+                self.connect_server(
                     server_id=srv.id,
                     name=srv.name,
                     transport=srv.transport,
@@ -427,9 +453,16 @@ class McpManager:
                     args=args,
                     env=env,
                     url=srv.url,
-                )
+                ),
-        finally:
+                timeout=20,
-            db.close()
+            )
+        except asyncio.TimeoutError:
+            logger.warning("Timed out connecting to %s", srv.name)
+            self._connections[srv.id] = {
+                "status": "timeout",
+                "error": f"Timed out after 20 seconds",
+                "name": srv.name,
+            }
 
     async def call_tool(self, qualified_name: str, arguments: Dict) -> Dict:
         """Call an MCP tool by its qualified name (mcp__{server_id}__{tool_name}).
@@ -504,7 +537,7 @@ class McpManager:
     async def _reconnect_builtin(self, server_id: str) -> bool:
         """Tear down and reconnect a crashed builtin MCP server."""
         import sys
-        from src.builtin_mcp import _BUILTIN_SERVERS
+        from src.builtin_mcp import _BUILTIN_SERVERS, builtin_python_env
 
         if server_id not in _BUILTIN_SERVERS:
             return False
@@ -523,7 +556,7 @@ class McpManager:
                 transport="stdio",
                 command=sys.executable,
                 args=[script_path],
-                env={"PYTHONPATH": base_dir},
+                env=builtin_python_env(base_dir),
             )
             if ok:
                 logger.info(f"Reconnected builtin MCP server: {name}")

src/mcp_oauth.py

@@ -96,7 +96,9 @@ class DbTokenStorage:
         try:
             srv = db.query(McpServer).filter(McpServer.id == self.server_id).first()
             if srv and srv.oauth_tokens:
-                return json.loads(srv.oauth_tokens)
+                parsed = json.loads(srv.oauth_tokens)
+                if isinstance(parsed, dict):
+                    return parsed
         finally:
             db.close()
         return {}
@@ -111,6 +113,8 @@ class DbTokenStorage:
             if srv is None:
                 return
             data = json.loads(srv.oauth_tokens) if srv.oauth_tokens else {}
+            if not isinstance(data, dict):
+                data = {}
             data[key] = value
             srv.oauth_tokens = json.dumps(data)
             db.commit()

src/rag_manager.py

@@ -32,9 +32,9 @@ class RAGManager:
         logger.info("RAGManager initialized as wrapper for VectorRAG")
     
     # Delegate all methods to VectorRAG
-    def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
+    def search(self, query: str, k: int = 5, owner: Optional[str] = None) -> List[Dict[str, Any]]:
         """Search for documents - delegates to VectorRAG."""
-        return self.vector_rag.search(query, k)
+        return self.vector_rag.search(query, k, owner=owner)
     
     def index_personal_documents(
         self,

src/task_action_policy.py

@@ -0,0 +1,47 @@
+"""Shared privilege policy for scheduled task actions."""
+
+from __future__ import annotations
+
+ADMIN_ONLY_TASK_ACTIONS = frozenset({
+    "run_local",
+    "run_script",
+    "ssh_command",
+    "cookbook_serve",
+})
+
+
+def is_admin_only_task_action(task_type: str | None, action: str | None) -> bool:
+    return (task_type or "llm") == "action" and (action or "") in ADMIN_ONLY_TASK_ACTIONS
+
+
+def owner_has_admin_task_privileges(owner: str | None) -> bool:
+    try:
+        from src.auth_helpers import _auth_disabled
+        if _auth_disabled():
+            return True
+    except Exception:
+        pass
+
+    if owner:
+        try:
+            from core.middleware import INTERNAL_TOOL_USER
+            if owner == INTERNAL_TOOL_USER:
+                return True
+        except Exception:
+            pass
+
+    try:
+        from core.auth import AuthManager
+        auth = AuthManager()
+        if not auth.is_configured:
+            return True
+        if not owner:
+            return False
+        return bool(auth.is_admin(owner))
+    except Exception:
+        pass
+
+    if not owner:
+        return False
+
+    return False

src/task_endpoint.py

@@ -74,4 +74,5 @@ async def task_llm_call_async(
     if not candidates:
         raise RuntimeError("No LLM endpoint available for background task")
     await wait_for_interactive_quiet("background task LLM")
+    kwargs.setdefault("workload", "background")
     return await llm_call_async_with_fallback(candidates, messages=messages, **kwargs)

src/task_scheduler.py

@@ -10,6 +10,10 @@ from datetime import datetime, timedelta, timezone
 from typing import Any, Awaitable, Callable, Dict, Tuple
 
 from core.auth import RESERVED_USERNAMES
+from src.task_action_policy import (
+    is_admin_only_task_action,
+    owner_has_admin_task_privileges,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -821,6 +825,28 @@ class TaskScheduler:
                     db.commit()
                 return
 
+            if (
+                is_admin_only_task_action(task.task_type, task.action)
+                and not owner_has_admin_task_privileges(task.owner)
+            ):
+                msg = f"Action '{task.action}' requires admin privileges"
+                blocked = db.query(TaskRun).filter(TaskRun.id == run_id).first()
+                if blocked:
+                    blocked.status = "error"
+                    blocked.result = msg
+                    blocked.error = msg
+                    blocked.finished_at = _utcnow()
+                task.status = "paused"
+                task.next_run = None
+                task.last_run = _utcnow()
+                logger.warning(
+                    "Paused admin-only task %s for non-admin owner %r",
+                    task_id,
+                    task.owner,
+                )
+                db.commit()
+                return
+
             if gate_foreground:
                 waiting = db.query(TaskRun).filter(TaskRun.id == run_id).first()
                 if waiting and waiting.status == "queued":
@@ -868,10 +894,10 @@ class TaskScheduler:
                     # Give the just-finished quiet gate a tiny grace window,
                     # then keep enforcing "background means background" while
                     # a long email/LLM action is already running.
-                    await asyncio.sleep(1.0)
+                    await asyncio.sleep(0.1)
                     from src.interactive_gate import has_foreground_activity
                     while True:
-                        await asyncio.sleep(1.0)
+                        await asyncio.sleep(0.25)
                         if has_foreground_activity():
                             foreground_cancel["hit"] = True
                             logger.info("Task '%s' interrupted because Odysseus became active", task.name)
@@ -1887,6 +1913,7 @@ class TaskScheduler:
             disabled_tools=disabled_tools,
             relevant_tools=relevant_tools,
             fallbacks=_task_fallbacks,
+            workload="background",
         ):
             if event_str.startswith("data: ") and not event_str.startswith("data: [DONE]"):
                 try:
@@ -2213,6 +2240,28 @@ class TaskScheduler:
         stopped = self._mark_run_aborted(task_id) or stopped
         return stopped
 
+    async def stop_background_tasks_for_foreground(self, *, reason: str = "Odysseus became active") -> int:
+        """Cancel all in-process scheduler tasks because the user is active.
+
+        This is intentionally blunt for scheduled/background work: when the
+        user opens or uses Odysseus, foreground interaction wins immediately.
+        Manual force-runs can be restarted by the user; automatic jobs will be
+        deferred by their cancellation path instead of stealing the app.
+        """
+        async with self._executing_lock:
+            task_ids = list(self._executing)
+        stopped = 0
+        for task_id in task_ids:
+            handle = self._task_handles.get(task_id)
+            if handle and not handle.done():
+                handle.cancel()
+                stopped += 1
+            if self._mark_run_aborted(task_id):
+                stopped += 1
+        if stopped:
+            logger.info("Stopped %d background scheduler task(s): %s", stopped, reason)
+        return stopped
+
     async def ensure_defaults(self, owner: str):
         """Create default housekeeping tasks for this owner (idempotent per action)."""
         from core.database import SessionLocal, ScheduledTask

src/tool_index.py

@@ -405,8 +405,10 @@ class ToolIndex:
             {"chat_with_model", "ask_teacher", "list_models"},
         # Deep research intent (incl. common typo "reserach")
         frozenset({"web search", "search the web", "search online", "look up",
-                   "google", "latest", "current", "news", "weather",
+                   "find info online", "find information online",
-                   "forecast", "stock price", "price of"}):
+                   "find info", "find information", "online about",
+                   "on the internet", "google", "latest", "current", "news",
+                   "weather", "forecast", "stock price", "price of"}):
             {"web_search", "web_fetch"},
         frozenset({"research", "reserach", "reasearch", "look into", "investigate",
                    "deep dive", "deep research", "find out about", "study up on",

src/tool_parsing.py

@@ -172,6 +172,27 @@ _GEMMA_TOOL_CALL_RE = re.compile(
     re.IGNORECASE,
 )
 
+# Pattern 4c: Open-function wrapper emitted by some local MLX/Exo models.
+# Example:
+#   <function_model>
+#   <function_call>web_search</function_call>
+#   <parameters>{"query":"Sweden news today"}</parameters>
+#   </function_model>
+_FUNCTION_MODEL_OPEN_RE = re.compile(r"<function_model>\s*", re.IGNORECASE)
+_FUNCTION_MODEL_CLOSE_RE = re.compile(r"</function_model>", re.IGNORECASE)
+_FUNCTION_MODEL_NAME_RE = re.compile(
+    r"<function_call>\s*([A-Za-z_][\w-]*)\s*</function_call>",
+    re.IGNORECASE,
+)
+_FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE)
+_FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE)
+_QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE)
+_QWEN_BARE_MARKER_RE = re.compile(
+    r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|"
+    r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)",
+    re.IGNORECASE,
+)
+
 
 # Pattern 5: DeepSeek DSML markup leaking into content. When deepseek
 # models can't emit structured tool_calls (e.g. we sent no tool schemas
@@ -566,6 +587,205 @@ def _parse_raw_web_json_lookup(text: str) -> Optional[tuple[ToolBlock, tuple[int
                 return block, (start, start + end)
     return None
 
+
+def _looks_like_openai_tool_call_blob(value) -> bool:
+    """Return True for raw OpenAI-style tool-call JSON leaked as text."""
+    if isinstance(value, list):
+        return bool(value) and all(_looks_like_openai_tool_call_blob(item) for item in value)
+    if not isinstance(value, dict):
+        return False
+    fn = value.get("function")
+    if isinstance(fn, dict) and isinstance(fn.get("name"), str):
+        return True
+    return False
+
+
+def _raw_openai_tool_call_to_block(value) -> Optional[ToolBlock]:
+    if isinstance(value, list):
+        for item in value:
+            block = _raw_openai_tool_call_to_block(item)
+            if block:
+                return block
+        return None
+    if not isinstance(value, dict):
+        return None
+    fn = value.get("function")
+    if not isinstance(fn, dict):
+        return None
+    name = str(fn.get("name") or "").strip()
+    if not name:
+        return None
+    tool_type = _TOOL_NAME_MAP.get(name, name)
+    raw_args = fn.get("arguments") or {}
+    try:
+        args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
+    except (json.JSONDecodeError, TypeError):
+        args = {}
+    if not isinstance(args, dict):
+        args = {}
+    # Common local-model typo seen in raw OpenAI JSON leaks.
+    if "text" not in args and "tex" in args:
+        args["text"] = args.get("tex")
+
+    if tool_type.startswith("mcp__"):
+        return ToolBlock(tool_type, json.dumps(args) if args else "{}")
+    if name in BUILTIN_EMAIL_TOOLS:
+        return ToolBlock(f"mcp__email__{name}", json.dumps(args) if args else "{}")
+    if tool_type not in TOOL_TAGS:
+        return None
+
+    if tool_type == "bash":
+        content = args.get("command", "")
+    elif tool_type == "python":
+        content = args.get("code", "")
+    elif tool_type == "web_search":
+        content = args.get("query", "")
+        queries = args.get("queries")
+        if not content and isinstance(queries, list) and queries:
+            content = str(queries[0])
+        elif not content and queries:
+            content = str(queries)
+        tf = args.get("time_filter")
+        if content and isinstance(tf, str) and tf in ("day", "week", "month", "year"):
+            content = json.dumps({"query": content, "time_filter": tf})
+    elif tool_type == "web_fetch":
+        content = args.get("url") or args.get("domain") or ""
+    elif tool_type == "read_file":
+        content = json.dumps(args) if (args.get("offset") or args.get("limit")) else args.get("path", "")
+    elif tool_type in ("grep", "glob", "ls", "edit_file"):
+        content = json.dumps(args) if args else "{}"
+    elif tool_type == "write_file":
+        content = args.get("path", "") + "\n" + args.get("content", "")
+    elif tool_type == "create_document":
+        parts = [args.get("title", "Untitled")]
+        if args.get("language"):
+            parts.append(args["language"])
+        parts.append(args.get("content", ""))
+        content = "\n".join(parts)
+    elif tool_type == "update_document":
+        content = args.get("content", "")
+    elif tool_type in ("edit_document", "suggest_document"):
+        marker = "SUGGEST" if tool_type == "suggest_document" else "REPLACE"
+        blocks = []
+        for edit in args.get("suggestions" if tool_type == "suggest_document" else "edits", []) or []:
+            if not isinstance(edit, dict):
+                continue
+            block = f'<<<FIND>>>\n{edit.get("find", "")}\n<<<{marker}>>>\n{edit.get("replace", "")}'
+            if tool_type == "suggest_document":
+                block += f'\n<<<REASON>>>\n{edit.get("reason", "")}'
+            blocks.append(block + "\n<<<END>>>")
+        content = "\n".join(blocks)
+    elif tool_type == "search_chats":
+        content = args.get("query", "")
+    elif tool_type == "chat_with_model":
+        content = args.get("model", "") + "\n" + args.get("message", "")
+    elif tool_type == "create_session":
+        content = args.get("name", "Untitled") + "\n" + args.get("model", "")
+    elif tool_type == "list_sessions":
+        content = args.get("filter", "")
+    elif tool_type == "send_to_session":
+        content = args.get("session_id", "") + "\n" + args.get("message", "")
+    elif tool_type == "pipeline":
+        content = json.dumps({"steps": args.get("steps", [])})
+    elif tool_type == "manage_session":
+        action = args.get("action", "")
+        if action == "list":
+            keyword = args.get("keyword", "") or args.get("value", "")
+            content = "list" + (("\n" + keyword) if keyword and keyword.lower() != "current" else "")
+        else:
+            content = action + "\n" + args.get("session_id", "current")
+            if args.get("value"):
+                content += "\n" + args["value"]
+    elif tool_type == "manage_memory":
+        action = args.get("action", "")
+        if action == "add":
+            content = "add\n" + str(args.get("text", ""))
+            if args.get("category"):
+                content += "\n" + str(args["category"])
+        elif action == "edit":
+            content = "edit\n" + str(args.get("memory_id", "")) + "\n" + str(args.get("text", ""))
+        elif action == "delete":
+            content = "delete\n" + str(args.get("memory_id", ""))
+        elif action == "search":
+            content = "search\n" + str(args.get("text", ""))
+        elif action == "list":
+            content = "list" + (("\n" + str(args["category"])) if args.get("category") else "")
+        else:
+            content = action
+    elif tool_type == "ui_control":
+        action = args.get("action", "")
+        name_arg = args.get("name", "")
+        value = args.get("value", "")
+        if action == "open_panel":
+            content = f"open_panel {name_arg or value}"
+        elif action == "toggle":
+            content = f"toggle {name_arg} {value}"
+        else:
+            content = action
+    elif tool_type in ("manage_tasks", "manage_skills", "api_call", "manage_endpoints",
+                       "manage_mcp", "manage_webhooks", "manage_tokens",
+                       "manage_documents", "manage_settings", "manage_notes",
+                       "manage_research", "manage_bg_jobs"):
+        content = json.dumps(args)
+    elif tool_type in ("get_workspace", "list_models"):
+        content = args.get("filter", "") if tool_type == "list_models" else ""
+    else:
+        content = json.dumps(args) if args else ""
+    return ToolBlock(tool_type, str(content or ""))
+
+
+def _parse_raw_openai_tool_call_json(text: str) -> Optional[ToolBlock]:
+    if not isinstance(text, str) or '"function"' not in text:
+        return None
+    decoder = json.JSONDecoder()
+    for match in re.finditer(r"[\[{]", text):
+        try:
+            parsed, _end = decoder.raw_decode(text[match.start():])
+        except json.JSONDecodeError:
+            continue
+        block = _raw_openai_tool_call_to_block(parsed)
+        if block:
+            return block
+    return None
+
+
+def _strip_raw_openai_tool_call_json(text: str) -> str:
+    """Strip raw JSON tool calls such as {"function": {...}, "type": "function"}.
+
+    Some local models emit native tool-call JSON into assistant text. The agent
+    can still parse/execute it through the native path, but the raw payload must
+    not render or persist as prose.
+    """
+    if not isinstance(text, str) or '"function"' not in text:
+        return text
+    decoder = json.JSONDecoder()
+    pieces = []
+    pos = 0
+    changed = False
+    for match in re.finditer(r"[\[{]", text):
+        start = match.start()
+        if start < pos:
+            continue
+        try:
+            parsed, rel_end = decoder.raw_decode(text[start:])
+        except json.JSONDecodeError:
+            continue
+        end = start + rel_end
+        if not _looks_like_openai_tool_call_blob(parsed):
+            continue
+        pieces.append(text[pos:start])
+        pos = end
+        changed = True
+        # Common broken local-model suffix: a standalone ] before a role marker.
+        while pos < len(text) and text[pos] in " \t\r\n":
+            pos += 1
+        if pos < len(text) and text[pos] == "]":
+            pos += 1
+    if not changed:
+        return text
+    pieces.append(text[pos:])
+    return "".join(pieces)
+
 def _parse_tool_call_block(raw: str) -> Optional[ToolBlock]:
     """Parse a [TOOL_CALL] block into a ToolBlock.
 
@@ -885,6 +1105,24 @@ def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
     return function_call_to_tool_block(tool_name, json.dumps(params))
 
 
+def _parse_function_model_call(body: str) -> Optional[ToolBlock]:
+    """Parse <function_model><function_call>tool</...><parameters>...</...>."""
+    name_match = _FUNCTION_MODEL_NAME_RE.search(body or "")
+    if not name_match:
+        return None
+    tool_name = name_match.group(1).strip().lower().replace("-", "_")
+    params = "{}"
+    for _ms, inner_start, inner_end, _me in _iter_delimited(
+        body,
+        _FUNCTION_MODEL_PARAMS_OPEN_RE,
+        _FUNCTION_MODEL_PARAMS_CLOSE_RE,
+    ):
+        params = body[inner_start:inner_end].strip() or "{}"
+        break
+    from src.tool_schemas import function_call_to_tool_block
+    return function_call_to_tool_block(tool_name, params)
+
+
 def _iter_delimited(text, open_re, close_re):
     """Yield ``(match_start, inner_start, inner_end, match_end)`` for each
     non-overlapping ``open_re ... close_re`` pair, scanning strictly forward.
@@ -1136,6 +1374,22 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]:
             if block:
                 blocks.append(block)
 
+    # Pattern 4c: <function_model> wrapper from local MLX/Exo models.
+    if not blocks:
+        for _ms, inner_start, inner_end, _me in _iter_delimited(
+            text, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE
+        ):
+            block = _parse_function_model_call(text[inner_start:inner_end])
+            if block:
+                blocks.append(block)
+
+    # Pattern 4d: raw OpenAI-style tool-call JSON leaked as assistant text.
+    # Example: {"function":{"arguments":"{\"action\":\"add\"}","name":"manage_memory"},"type":"function"}
+    if not blocks:
+        block = _parse_raw_openai_tool_call_json(text)
+        if block:
+            blocks.append(block)
+
     # Pattern 6: local text-model web_search call leaked as prose + bare JSON.
     if not blocks and not skip_fenced:
         raw_web_json = _parse_raw_web_json_lookup(text)
@@ -1182,6 +1436,10 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str:
     cleaned = _XML_OPEN_TOOL_CALL_RE.sub('', cleaned)
     cleaned = _strip_delimited(cleaned, _TOOL_CODE_OPEN_RE, _TOOL_CODE_CLOSE_RE)
     cleaned = _GEMMA_TOOL_CALL_RE.sub('', cleaned)
+    cleaned = _strip_delimited(cleaned, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE)
+    cleaned = _strip_raw_openai_tool_call_json(cleaned)
+    cleaned = _QWEN_ROLE_MARKER_RE.sub('', cleaned)
+    cleaned = _QWEN_BARE_MARKER_RE.sub(' ', cleaned)
     if not skip_fenced:
         raw_web_json = _parse_raw_web_json_lookup(cleaned)
         if raw_web_json:

src/tool_policy.py

@@ -16,6 +16,39 @@ GUIDE_ONLY_DIRECTIVE = (
     "output they will produce locally."
 )
 
+WEB_TOOL_NAMES = frozenset({"web_search", "web_fetch"})
+
+
+def tool_toggle_enabled(value: object) -> bool:
+    """Return true only for explicit true-like tool toggle values."""
+
+    return str(value).lower() == "true"
+
+
+def tool_toggle_explicitly_denied(value: object) -> bool:
+    """Return true when a caller explicitly supplied a non-true toggle value."""
+
+    return value is not None and not tool_toggle_enabled(value)
+
+
+def is_web_search_explicitly_denied(allow_web_search: object) -> bool:
+    """Whether the web-search agent toggle was explicitly set to false."""
+
+    return tool_toggle_explicitly_denied(allow_web_search)
+
+
+def web_search_enabled_for_turn(allow_web_search: object, use_web: object = None) -> bool:
+    """Return true only when this request explicitly enables web search.
+
+    Agent mode sends ``allow_web_search``; chat-mode pre-search sends
+    ``use_web``. If both are present, an explicit ``allow_web_search=false``
+    wins so a stale or conflicting intent path cannot re-enable web tools.
+    """
+
+    if is_web_search_explicitly_denied(allow_web_search):
+        return False
+    return tool_toggle_enabled(allow_web_search) or tool_toggle_enabled(use_web)
+
 
 _COMMON_TOOL_NAMES = {
     "api_call",

src/tool_schemas.py

@@ -18,6 +18,15 @@ from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
+
+_REQUIRED_NATIVE_TOOL_ARGS = {
+    "web_search": ("query", "queries"),
+    "web_fetch": ("url",),
+    "read_file": ("path",),
+    "write_file": ("path",),
+    "edit_file": ("path",),
+}
+
 # ---------------------------------------------------------------------------
 # OpenAI-compatible function tool schemas
 # ---------------------------------------------------------------------------
@@ -187,7 +196,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "create_document",
-            "description": "Create a new document in the editor panel. Use this when the user asks to write, create, build, or generate code, scripts, programs, games, apps, or any substantial content (>15 lines) AND there is no already-open document/email draft that the request refers to. If an email compose draft is open, edit that draft instead of creating another document. NEVER put large code blocks directly in chat — use this tool instead.",
+            "description": "Create a new document in the editor panel. Use this when the user asks to write, create, build, make, or generate code, scripts, programs, games, apps, or any long-form or structured content that is more than a short paragraph, AND there is no already-open document/email draft that the request refers to. If an email compose draft is open, edit that draft instead of creating another document. NEVER put large generated content directly in chat — use this tool instead.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -556,8 +565,8 @@ FUNCTION_TOOL_SCHEMAS = [
                     "uid": {"type": "string", "description": "Event UID (for update/delete)"},
                     "calendar_href": {"type": "string", "description": "Specific calendar URL (optional; defaults to first calendar)"},
                     "calendar": {"type": "string", "description": "Filter list_events by calendar name or href"},
-                    "start": {"type": "string", "description": "list_events range start (ISO datetime); defaults to today. Prefer start; backend also accepts start_date, range_start, from, dtstart, since."},
+                    "start": {"type": "string", "description": "list_events range start (ISO datetime). Use this for month/week requests after resolving the date range; do not pass a loose query string. Prefer start; backend also accepts start_time, start_date, range_start, from, dtstart, since."},
-                    "end": {"type": "string", "description": "list_events range end (ISO datetime); defaults to +14 days. Prefer end; backend also accepts end_date, range_end, to, dtend, until."},
+                    "end": {"type": "string", "description": "list_events range end (ISO datetime). Use this for month/week requests after resolving the date range; defaults to +14 days only when no range is requested. Prefer end; backend also accepts end_time, end_date, range_end, to, dtend, until."},
                     "event_type": {"type": "string", "description": "Tag / category for the event. Common values: work, personal, health, travel, meal, social, admin, other. Aliases accepted: tag, category, type."},
                     "importance": {"type": "string", "enum": ["low", "normal", "high", "critical"], "description": "Priority level (defaults to 'normal')"},
                     "reminder_minutes": {"type": "integer", "description": "For create_event: create an Odysseus reminder this many minutes before the event, e.g. 5 for 'reminder 5 min before'."},
@@ -576,9 +585,10 @@ FUNCTION_TOOL_SCHEMAS = [
                 "type": "object",
                 "properties": {
                     "action": {"type": "string",
-                               "enum": ["list", "view", "add", "update", "delete", "toggle_item"],
+                               "enum": ["list", "search", "view", "add", "update", "delete", "toggle_item"],
                                "description": "The action to perform"},
                     "id": {"type": "string", "description": "Note id (for update/delete/toggle_item); 8-char prefix is fine"},
+                    "query": {"type": "string", "description": "Search text for action='search'"},
                     "title": {"type": "string", "description": "Note title (for add/update)"},
                     "content": {"type": "string", "description": "Freeform body text. Use this for note_type='note'. Do NOT use this for checklists — pass `checklist_items` instead."},
                     "note_type": {"type": "string", "enum": ["note", "checklist"],
@@ -1025,7 +1035,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "manage_contact",
-            "description": "Create, update, delete, or list the user's CardDAV contacts. Use to save a new contact, update an existing one (email/phone/address), or remove one. For update/delete you need the contact's uid — call action='list' first to find it. Writes go through the same dedupe + validation as the Contacts UI.",
+            "description": "Create, update, delete, or list the user's CardDAV contacts. Use to save a new contact, update an existing one (email/phone/address), or remove one. Add does not require email: name + phone or name + address is valid. For update/delete you need the contact's uid — call action='list' first to find it. Writes go through the same dedupe + validation as the Contacts UI.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -1033,9 +1043,9 @@ FUNCTION_TOOL_SCHEMAS = [
                                "description": "list = show all contacts (with uids); add = create; update = edit by uid; delete = remove by uid."},
                     "uid": {"type": "string", "description": "Contact UID (required for update/delete; get it from action=list)."},
                     "name": {"type": "string", "description": "Contact's display name (for add/update)."},
-                    "email": {"type": "string", "description": "Single email address (convenience for add, or the primary email for update)."},
+                    "email": {"type": "string", "description": "Single email address (convenience for add, or the primary email for update). Optional when phone or address is provided."},
-                    "emails": {"type": "array", "items": {"type": "string"}, "description": "Full list of email addresses (for update; first is primary)."},
+                    "emails": {"type": "array", "items": {"type": "string"}, "description": "Full list of email addresses (first is primary)."},
-                    "phones": {"type": "array", "items": {"type": "string"}, "description": "Full list of phone numbers (for update)."},
+                    "phones": {"type": "array", "items": {"type": "string"}, "description": "Full list of phone numbers. Valid for add/update."},
                     "address": {"type": "string", "description": "Postal/mailing address as a single human-readable string."},
                 },
                 "required": ["action"]
@@ -1308,6 +1318,11 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
         logger.warning(f"Non-object function call arguments for {name}: {args!r}; treating as empty")
         args = {}
 
+    required_args = _REQUIRED_NATIVE_TOOL_ARGS.get(tool_type)
+    if required_args and not any(str(args.get(key) or "").strip() for key in required_args):
+        logger.warning(f"Rejecting empty required arguments for function call {name}: {args!r}")
+        return None
+
     # Allow MCP tools through (namespaced as mcp__serverid__toolname)
     if tool_type.startswith("mcp__"):
         content = json.dumps(args) if args else "{}"
@@ -1415,15 +1430,20 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
     elif tool_type == "manage_memory":
         action = args.get("action", "")
         if action == "add":
-            content = "add\n" + args.get("text", "")
+            text = args.get("text") or args.get("value") or args.get("content") or ""
+            if not text and args.get("key"):
+                text = str(args.get("key") or "")
+            content = "add\n" + str(text)
             if args.get("category"):
                 content += "\n" + args["category"]
+            elif args.get("key"):
+                content += "\n" + str(args["key"])
         elif action == "edit":
             content = "edit\n" + args.get("memory_id", "") + "\n" + args.get("text", "")
         elif action == "delete":
             content = "delete\n" + args.get("memory_id", "")
         elif action == "search":
-            content = "search\n" + args.get("text", "")
+            content = "search\n" + (args.get("text") or args.get("tex") or args.get("query") or "")
         elif action == "list":
             content = "list"
             if args.get("category"):

src/tool_utils.py

@@ -9,6 +9,7 @@ import json
 from src.constants import MAX_OUTPUT_CHARS
 
 _mcp_manager = None
+_upload_handler = None
 
 # ---------------------------------------------------------------------------
 # MCP Manager singleton
@@ -23,6 +24,21 @@ def get_mcp_manager():
     """Get the global MCP manager instance."""
     return _mcp_manager
 
+
+# ---------------------------------------------------------------------------
+# Shared upload lifecycle handler
+# ---------------------------------------------------------------------------
+
+def set_upload_handler(handler):
+    """Register the process's UploadHandler without importing app modules."""
+    global _upload_handler
+    _upload_handler = handler
+
+
+def get_upload_handler():
+    """Return the shared UploadHandler used by route and agent writers."""
+    return _upload_handler
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -54,6 +70,8 @@ def _parse_tool_args(content):
     if isinstance(content, str):
         try:
             args = json.loads(content) if content.strip() else {}
+            if not isinstance(args, dict):
+                args = {}
         except (json.JSONDecodeError, TypeError) as e:
             raise ValueError(str(e))
     elif isinstance(content, dict):

src/tools/calendar.py

@@ -10,6 +10,8 @@ import re
 from typing import Dict, Optional
 
 from src.tools._common import _parse_tool_args
+from src.tool_utils import get_upload_handler
+from src.upload_handler import reserve_upload_references
 
 logger = logging.getLogger(__name__)
 
@@ -208,11 +210,20 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
         elif action == "list_events":
             try:
                 start_raw = _first_nonempty_arg(
-                    "start", "start_date", "range_start", "from", "dtstart", "since"
+                    "start", "start_time", "start_date", "range_start", "from", "dtstart", "since"
                 )
                 end_raw = _first_nonempty_arg(
-                    "end", "end_date", "range_end", "to", "dtend", "until"
+                    "end", "end_time", "end_date", "range_end", "to", "dtend", "until"
                 )
+                query_raw = args.get("query") or args.get("date_range") or args.get("range")
+                if query_raw and (not start_raw or not end_raw):
+                    return {
+                        "error": (
+                            "list_events needs explicit start/end ISO datetimes; "
+                            f"resolve the requested range ({query_raw!r}) and call manage_calendar again."
+                        ),
+                        "exit_code": 1,
+                    }
                 if start_raw:
                     start_dt = _parse_dt(start_raw)
                 else:
@@ -399,11 +410,25 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
             importance = args.get("importance") or "normal"
             minutes_before = _reminder_minutes(args)
 
+            event_description = _event_description(args, minutes_before)
+            event_location = args.get("location", "") or ""
+            missing_id = reserve_upload_references(
+                get_upload_handler(),
+                owner,
+                event_description,
+                event_location,
+            )
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
+
             uid = str(_uuid.uuid4())
             ev = CalendarEvent(
                 uid=uid, calendar_id=cal.id, summary=summary,
-                description=_event_description(args, minutes_before),
+                description=event_description,
-                location=args.get("location", "") or "",
+                location=event_location,
                 dtstart=dtstart, dtend=dtend, all_day=all_day,
                 is_utc=dtstart_is_utc and not all_day,
                 rrule=args.get("rrule", "") or "",
@@ -456,6 +481,17 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
             ev = _event_query().filter(CalendarEvent.uid == base_uid).first()
             if not ev:
                 return {"error": f"Event {uid} not found", "exit_code": 1}
+            missing_id = reserve_upload_references(
+                get_upload_handler(),
+                owner,
+                args.get("description"),
+                args.get("location"),
+            )
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
             if args.get("summary") is not None:
                 ev.summary = args["summary"]
             if args.get("description") is not None:

src/tools/contacts.py

@@ -108,16 +108,28 @@ async def do_manage_contact(content: str, owner: Optional[str] = None) -> Dict:
 
         if action == "add":
             email = (args.get("email") or "").strip()
-            if not email:
+            phones = [str(p or "").strip() for p in (args.get("phones") or []) if str(p or "").strip()]
-                return {"error": "email is required for add", "exit_code": 1}
+            phone = (args.get("phone") or "").strip()
-            name = (args.get("name") or "").strip() or email.split("@")[0]
+            if phone and phone not in phones:
-            # Dedupe by email (same as the /add route).
+                phones.insert(0, phone)
+            address = (args.get("address") or "").strip()
+            name = (args.get("name") or "").strip()
+            if not name and email:
+                name = email.split("@")[0]
+            if not name and not email and not phones and not address:
+                return {"error": "name plus email, phone, or address is required for add", "exit_code": 1}
+            if not name:
+                name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
+            # Dedupe by email or phone (same as the /add route).
             existing = await asyncio.to_thread(cc._fetch_contacts)
             for c in existing:
-                if email.lower() in [e.lower() for e in c.get("emails", [])]:
+                if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
                     return {"output": f"{email} is already a contact ({c.get('name','')}).", "exit_code": 0}
-            ok = await asyncio.to_thread(cc._create_contact, name, email)
+                if phones and any(p in (c.get("phones") or []) for p in phones):
-            return {"output": f"{'Added' if ok else 'Failed to add'} {name} <{email}>.", "exit_code": 0 if ok else 1}
+                    return {"output": f"{phones[0]} is already a contact ({c.get('name','')}).", "exit_code": 0}
+            ok = await asyncio.to_thread(cc._create_contact, name, email, address, phones)
+            detail = email or ", ".join(phones) or address
+            return {"output": f"{'Added' if ok else 'Failed to add'} {name} ({detail}).", "exit_code": 0 if ok else 1}
 
         if action in ("update", "edit"):
             uid = (args.get("uid") or "").strip()
@@ -129,11 +141,12 @@ async def do_manage_contact(content: str, owner: Optional[str] = None) -> Dict:
                 emails = [args["email"]]
             emails = [e.strip() for e in (emails or []) if e and e.strip()]
             phones = [p.strip() for p in (args.get("phones") or []) if p and p.strip()]
-            if not name and not emails:
+            address = (args.get("address") or "").strip()
-                return {"error": "Provide a name or emails to update", "exit_code": 1}
+            if not name and not emails and not phones and not address:
+                return {"error": "Provide a name, emails, phones, or address to update", "exit_code": 1}
             if not name and emails:
                 name = emails[0].split("@")[0]
-            ok = await asyncio.to_thread(cc._update_contact, uid, name, emails, phones)
+            ok = await asyncio.to_thread(cc._update_contact, uid, name, emails, phones, address)
             return {"output": "Contact updated." if ok else "Update failed.", "exit_code": 0 if ok else 1}
 
         if action == "delete":

src/tools/cookbook.py

@@ -315,6 +315,7 @@ async def _cookbook_register_task(
 _MODEL_PROCESS_PATTERNS = [
     ("vLLM",            ["vllm.entrypoints", "vllm serve", "/vllm/", "vllm-openai"]),
     ("SGLang",          ["sglang.launch_server", "sglang/launch_server"]),
+    ("MLX",             ["mlx_lm.server", "mlx-lm"]),
     ("llama.cpp",       ["llama-server", "llama_cpp_server", "llamacppserver"]),
     ("Ollama",          ["ollama serve", "ollama runner", "/ollama "]),
     ("ComfyUI",         ["comfyui/main.py", "/ComfyUI/main.py", "ComfyUI"]),
@@ -590,7 +591,7 @@ async def do_serve_model(content: str, owner: Optional[str] = None) -> Dict:
         hint = ""
         if isinstance(err_msg, str) and "cmd" in err_msg.lower():
             hint = (" — the cmd must START with an allowlisted binary "
-                    "(vllm, python3, llama-server, ollama, sglang, lmdeploy, node, npx). "
+                    "(vllm, python3, llama-server, ollama, sglang, mlx_lm, lmdeploy, node, npx). "
                     "Do NOT prefix with `cd …`, `source …`, or chain with `&&`. "
                     "env_prefix (e.g. `source ~/qwen35-env/bin/activate`) is added "
                     "automatically from the host's saved venv settings.")
@@ -635,7 +636,7 @@ async def do_list_served_models(content: str, owner: Optional[str] = None) -> Di
 
     if not merged:
         return {
-            "output": "No model servers currently running (cookbook task tracker empty; /proc scan found no vLLM / sglang / llama.cpp / Ollama / ComfyUI / A1111 / Fooocus / InvokeAI / TGI / Aphrodite / Triton / Diffusers processes).",
+            "output": "No model servers currently running (cookbook task tracker empty; /proc scan found no vLLM / sglang / MLX / llama.cpp / Ollama / ComfyUI / A1111 / Fooocus / InvokeAI / TGI / Aphrodite / Triton / Diffusers processes).",
             "exit_code": 0,
         }
 

src/tools/notes.py

@@ -10,6 +10,8 @@ import re
 from typing import Dict, Optional
 
 from src.tools._common import _parse_tool_args
+from src.tool_utils import get_upload_handler
+from src.upload_handler import reserve_upload_references
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +29,8 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
 
     # Action aliases — match what models actually emit. `create` is the most
     # common alternative to `add`. Hyphenated forms also accepted.
-    action = (args.get("action") or "").replace("-", "_").strip().lower()
+    raw_action = (args.get("action") or "").replace("-", "_").strip().lower()
+    action = raw_action
     _NOTE_ACTION_ALIASES = {
         "create": "add",
         "new": "add",
@@ -60,37 +63,68 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
             q = q.filter(Note.owner == owner)
         return q.first()
 
+    def _format_note_list(notes) -> str:
+        lines = []
+        for n in notes:
+            pin = " [PINNED]" if n.pinned else ""
+            typ = " [checklist]" if n.note_type == "checklist" else ""
+            lbl = f" #{n.label}" if n.label else ""
+            title = n.title or "(untitled)"
+            lines.append(f"- [{n.id[:8]}] **{title}**{pin}{typ}{lbl}")
+            if n.note_type == "checklist" and n.items:
+                try:
+                    items = json.loads(n.items)
+                    for i, item in enumerate(items):
+                        mark = "x" if item.get("done") else " "
+                        lines.append(f"  [{mark}] {i}: {item.get('text', '')}")
+                except (json.JSONDecodeError, TypeError):
+                    pass
+            elif n.content:
+                snippet = n.content[:80].replace("\n", " ")
+                lines.append(f"  {snippet}")
+        return "\n".join(lines)
+
     try:
-        if action == "list":
+        if action in ("list", "search", "find"):
             q = db.query(Note)
             if owner is not None:
                 q = q.filter(Note.owner == owner)
-            if args.get("label"):
+            label_filter = str(args.get("label") or "").strip()
-                q = q.filter(Note.label == args["label"])
+            if label_filter and label_filter.lower() != "default":
+                q = q.filter(Note.label == label_filter)
             show_archived = args.get("archived", False)
             q = q.filter(Note.archived == show_archived)
             notes = q.order_by(Note.pinned.desc(), Note.updated_at.desc()).all()
+            if action in ("search", "find"):
+                query = str(
+                    args.get("query")
+                    or args.get("text")
+                    or args.get("title")
+                    or args.get("content")
+                    or ""
+                ).strip().lower()
+                if query:
+                    filtered = []
+                    for n in notes:
+                        haystack = " ".join(
+                            str(part or "")
+                            for part in (n.title, n.content, n.label, n.items)
+                        ).lower()
+                        if query in haystack:
+                            filtered.append(n)
+                    notes = filtered
             if not notes:
                 return {"response": "No notes found.", "exit_code": 0}
-            lines = []
+            return {"results": _format_note_list(notes), "exit_code": 0}
-            for n in notes:
+
-                pin = " [PINNED]" if n.pinned else ""
+        elif action == "view":
-                typ = " [checklist]" if n.note_type == "checklist" else ""
+            note_id = args.get("id", "")
-                lbl = f" #{n.label}" if n.label else ""
+            note = _note_by_prefix(note_id)
-                title = n.title or "(untitled)"
+            if not note:
-                lines.append(f"- [{n.id[:8]}] **{title}**{pin}{typ}{lbl}")
+                return {"error": f"Note '{note_id}' not found", "exit_code": 1}
-                if n.note_type == "checklist" and n.items:
+            if not _note_visible_to_owner(note, owner):
-                    try:
+                return {"error": "Note not found", "exit_code": 1}
-                        items = json.loads(n.items)
+            return {"results": _format_note_list([note]), "exit_code": 0}
-                        for i, item in enumerate(items):
-                            mark = "x" if item.get("done") else " "
-                            lines.append(f"  [{mark}] {i}: {item.get('text', '')}")
-                    except (json.JSONDecodeError, TypeError):
-                        pass
-                elif n.content:
-                    snippet = n.content[:80].replace("\n", " ")
-                    lines.append(f"  {snippet}")
-            return {"results": "\n".join(lines)}
 
         elif action == "add":
             # Accept the various field names models emit: `text` is the most
@@ -120,6 +154,25 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
             # `new Date()` resolves the right absolute moment regardless of
             # where the user is.
             due_raw = args.get("due_date")
+            if not due_raw:
+                combined_text = " ".join(
+                    str(v or "")
+                    for v in (title, content_raw, text_raw)
+                ).strip()
+                lower_combined = combined_text.lower()
+                looks_like_reminder = (
+                    raw_action in {"remind", "reminder"}
+                    or re.search(r"\bremind(?:er)?\b", lower_combined)
+                )
+                if looks_like_reminder:
+                    temporal = re.search(
+                        r"\b(?:today|tonight|tomorrow|tmrw|yesterday)\b(?:\s+(?:at\s+)?\d{1,2}(?::\d{2})?\s*(?:am|pm)?)?"
+                        r"|\b\d{1,2}(?::\d{2})?\s*(?:am|pm)?\s+(?:today|tonight|tomorrow|tmrw|yesterday)\b"
+                        r"|\bin\s+\d+\s*(?:hour|hr|minute|min|day)s?\b",
+                        lower_combined,
+                    )
+                    if temporal:
+                        due_raw = temporal.group(0)
             due_iso = None
             if due_raw:
                 try:
@@ -147,6 +200,18 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
                             "duplicate": True,
                             "exit_code": 0,
                         }
+            missing_id = reserve_upload_references(
+                get_upload_handler(),
+                owner,
+                content_raw,
+                args.get("color"),
+                items_json,
+            )
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
             note = Note(
                 id=str(_uuid.uuid4()),
                 owner=owner,
@@ -170,7 +235,7 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
             # link with no target, leaving the user with a click that
             # did nothing and uncertainty about whether the note was made.
             return {
-                "response": f"Note created: \"{title or '(untitled)'}\" (id: {note.id[:8]})",
+                "response": f"{'Reminder' if due_iso else 'Note'} created: \"{title or '(untitled)'}\" (id: {note.id[:8]})",
                 "note_id": note.id,
                 "note_title": title or "",
                 "open_url": f"/#open=notes&note={note.id}",
@@ -184,6 +249,19 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
                 return {"error": f"Note '{note_id}' not found", "exit_code": 1}
             if not _note_visible_to_owner(note, owner):
                 return {"error": "Note not found", "exit_code": 1}
+            missing_id = reserve_upload_references(
+                get_upload_handler(),
+                owner,
+                args.get("content"),
+                args.get("color"),
+                args.get("checklist_items"),
+                args.get("items"),
+            )
+            if missing_id:
+                return {
+                    "error": f"Referenced upload is no longer available: {missing_id}",
+                    "exit_code": 1,
+                }
             for field in ("title", "content", "note_type", "color", "label"):
                 if field in args and args[field] is not None:
                     setattr(note, field, args[field])
@@ -246,7 +324,7 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
             return {"response": f"Item '{items[index].get('text', '')}' marked {mark}", "exit_code": 0}
 
         else:
-            return {"error": f"Unknown action: {action}. Use list/add/update/delete/toggle_item", "exit_code": 1}
+            return {"error": f"Unknown action: {action}. Use list/search/view/add/update/delete/toggle_item", "exit_code": 1}
     except Exception as e:
         logger.error(f"manage_notes error: {e}")
         return {"error": str(e), "exit_code": 1}

src/tools/search.py

@@ -34,8 +34,8 @@ async def do_search_chats(query: str, limit: int = 20, owner: str | None = None)
 
         lines = [f"Found {len(seen_sessions)} session(s) matching \"{query}\":\n"]
         for sid, result in seen_sessions.items():
-            lines.append(f"- **{result.session_name}** (#{sid})")
+            lines.append(f"- [**{result.session_name}**](#session-{sid})")
-            lines.append(f"  Link: [Open chat](#{sid})")
+            lines.append(f"  Open: [Open chat](#session-{sid})")
             lines.append(f"  Match ({result.role}): {result.content_snippet}")
             if result.context_before:
                 before = result.context_before[-1]

src/tools/system.py

@@ -356,7 +356,11 @@ async def do_manage_tasks(content: str, owner: Optional[str] = None) -> Dict:
             task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
             if not task:
                 return {"error": f"Task {task_id} not found", "exit_code": 1}
-            if owner and task.owner and task.owner != owner:
+            # Strict ownership: the old `task.owner and task.owner != owner`
+            # skipped the check on an owner-less task (created in no-login mode
+            # or before the legacy-owner sweep), letting any authenticated user
+            # reach it. `list` already scopes to an exact owner match.
+            if owner and task.owner != owner:
                 return {"error": "Access denied", "exit_code": 1}
 
             changed = []
@@ -402,7 +406,7 @@ async def do_manage_tasks(content: str, owner: Optional[str] = None) -> Dict:
             task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
             if not task:
                 return {"error": f"Task {task_id} not found", "exit_code": 1}
-            if owner and task.owner and task.owner != owner:
+            if owner and task.owner != owner:
                 return {"error": "Access denied", "exit_code": 1}
             name = task.name
             db.delete(task)
@@ -416,7 +420,7 @@ async def do_manage_tasks(content: str, owner: Optional[str] = None) -> Dict:
             task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
             if not task:
                 return {"error": f"Task {task_id} not found", "exit_code": 1}
-            if owner and task.owner and task.owner != owner:
+            if owner and task.owner != owner:
                 return {"error": "Access denied", "exit_code": 1}
 
             if action == "pause":
@@ -437,7 +441,7 @@ async def do_manage_tasks(content: str, owner: Optional[str] = None) -> Dict:
             task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
             if not task:
                 return {"error": f"Task {task_id} not found", "exit_code": 1}
-            if owner and task.owner and task.owner != owner:
+            if owner and task.owner != owner:
                 return {"error": "Access denied", "exit_code": 1}
 
             from src.event_bus import get_task_scheduler
@@ -538,6 +542,11 @@ _APP_API_BLOCKLIST_METHOD_PATH = (
     # sidebar surfaces the session. Raw start works but the agent
     # fumbles the payload + the session doesn't reliably show up.
     ("POST",   "/api/research/start"),
+    # Use web_search — the HTTP search route is UI-shaped and generic
+    # app_api calls can return empty/poorly formatted results compared with the
+    # named tool's source-aware output.
+    ("GET",    "/api/search"),
+    ("POST",   "/api/search"),
     # Use the named tools — they handle owner attribution, natural-
     # language due_date parsing, timezone, dedup, and tag/category
     # normalization. Hitting the raw endpoint via app_api saves a
@@ -653,6 +662,8 @@ async def do_app_api(content: str, owner: Optional[str] = None) -> Dict:
             return {"error": "Don't POST /api/model/serve directly — use the `serve_model` or `serve_preset` tool (handles host resolution, env_prefix, and cookbook tracking).", "exit_code": 1}
         if "/api/research/start" in path:
             return {"error": "Don't POST /api/research/start directly — use the `trigger_research` tool (it surfaces the session in the Deep Research sidebar).", "exit_code": 1}
+        if "/api/search" in path:
+            return {"error": "Don't hit /api/search via app_api — use the `web_search` tool for online lookups, or `web_fetch` for a specific URL.", "exit_code": 1}
         if "/api/notes" in path:
             return {"error": "Don't hit /api/notes via app_api — use the `manage_notes` tool. It accepts natural-language due_date ('11pm today', 'tomorrow at 9am'), fires reminders from the due_date itself (no separate calendar event), and uses the caller's timezone. The raw endpoint requires ISO-UTC + a separate calendar event, both of which the agent tends to get wrong.", "exit_code": 1}
         if "/api/calendar/events" in path:

src/upload_handler.py

@@ -35,11 +35,34 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+
+class UploadCleanupSafetyError(RuntimeError):
+    """Raised when cleanup cannot prove that destructive work is safe."""
+
 # The extension is optional: save_upload builds the id as `{uuid.hex}{ext}`,
 # and a file with no extension (Dockerfile, README, ...) yields a bare 32-hex
 # id. Requiring `.ext` made those ids fail validation, so the stored file
 # could never be resolved or downloaded again.
 UPLOAD_ID_RE = re.compile(r"^[0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?$")
+UPLOAD_ID_TOKEN_RE = re.compile(
+    r"(?<![0-9a-fA-F])([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)(?![A-Za-z0-9])"
+)
+INTERNAL_UPLOAD_URL_RE = re.compile(
+    r"(?:odysseus://attachment/|/api/upload/)"
+    r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
+    r"(?=$|[\s\"'<>\[\](){},;!?:&#]|\.(?![A-Za-z0-9]))"
+)
+PDF_SOURCE_UPLOAD_RE = re.compile(
+    r"<!--\s*pdf(?:_form)?_source\b[^>]*\bupload_id="
+    r"[\"']([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)[\"'][^>]*-->",
+    re.IGNORECASE,
+)
+ATTACHMENT_REFERENCE_LINE_RE = re.compile(
+    r"\[Attachment:[^\]\r\n]*\|\s*id="
+    r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
+    r"(?:\s*\||\s*\])",
+    re.IGNORECASE,
+)
 
 
 def is_valid_upload_id(upload_id: str) -> bool:
@@ -47,6 +70,110 @@ def is_valid_upload_id(upload_id: str) -> bool:
     return UPLOAD_ID_RE.fullmatch(upload_id or "") is not None
 
 
+def extract_upload_ids(value: Any) -> set[str]:
+    """Return canonical upload IDs embedded in a persisted URL/text value."""
+    if not isinstance(value, str) or not value:
+        return set()
+    return set(UPLOAD_ID_TOKEN_RE.findall(value))
+
+
+def extract_internal_upload_ids(value: Any) -> set[str]:
+    """Return IDs from explicit internal upload references only.
+
+    Cleanup intentionally uses :func:`extract_upload_ids` conservatively, but
+    write-time reservation must not treat an arbitrary 32-hex checksum in note
+    or calendar text as an upload reference. Nested JSON-like values are
+    supported because note checklist items are persisted as structured data.
+    """
+    if isinstance(value, dict):
+        found: set[str] = set()
+        for nested in value.values():
+            found.update(extract_internal_upload_ids(nested))
+        return found
+    if isinstance(value, (list, tuple, set)):
+        found: set[str] = set()
+        for nested in value:
+            found.update(extract_internal_upload_ids(nested))
+        return found
+    if not isinstance(value, str) or not value:
+        return set()
+    return (
+        set(INTERNAL_UPLOAD_URL_RE.findall(value))
+        | set(PDF_SOURCE_UPLOAD_RE.findall(value))
+        | set(ATTACHMENT_REFERENCE_LINE_RE.findall(value))
+    )
+
+
+def reserve_upload_references(
+    upload_handler: Any,
+    owner: Optional[str],
+    *values: Any,
+) -> Optional[str]:
+    """Reserve upload IDs in values before a caller persists references.
+
+    Returns the first ID that cannot be owner-checked/reserved, otherwise
+    ``None``. A missing handler is treated as no-op for backward-compatible
+    route factories; production wires the shared UploadHandler instance.
+    """
+    if upload_handler is None:
+        return None
+    upload_ids: set[str] = set()
+    for value in values:
+        upload_ids.update(extract_internal_upload_ids(value))
+    return reserve_upload_ids(upload_handler, owner, upload_ids)
+
+
+def reserve_upload_ids(
+    upload_handler: Any,
+    owner: Optional[str],
+    upload_ids: Any,
+) -> Optional[str]:
+    """Owner-reserve canonical IDs from a trusted structured reference field."""
+    if upload_handler is None:
+        return None
+    canonical_ids = {
+        str(upload_id).strip()
+        for upload_id in (upload_ids or [])
+        if is_valid_upload_id(str(upload_id).strip())
+    }
+    for upload_id in sorted(canonical_ids):
+        try:
+            resolved = upload_handler.reserve_upload(
+                upload_id,
+                owner=owner,
+                allow_admin=False,
+            )
+        except Exception:
+            resolved = None
+        if not resolved:
+            return upload_id
+    return None
+
+
+def reserve_message_upload_references(
+    upload_handler: Any,
+    owner: Optional[str],
+    content: Any,
+    metadata: Any = None,
+) -> Optional[str]:
+    """Reserve explicit chat references, including structured attachment IDs."""
+    upload_ids = extract_internal_upload_ids(content)
+    if metadata not in (None, ""):
+        if isinstance(metadata, str):
+            metadata = json.loads(metadata)
+        if not isinstance(metadata, dict):
+            raise ValueError("message metadata must be a JSON object")
+        upload_ids.update(extract_internal_upload_ids(metadata))
+        from src.attachment_refs import attachment_refs_from_metadata
+
+        upload_ids.update(
+            str(ref.get("attachment_id") or "").strip()
+            for ref in attachment_refs_from_metadata(metadata)
+            if ref.get("attachment_id")
+        )
+    return reserve_upload_ids(upload_handler, owner, upload_ids)
+
+
 def _build_upload_id(safe_filename: str) -> str:
     """Build a unique upload id whose extension matches UPLOAD_ID_RE.
 
@@ -249,43 +376,295 @@ class UploadHandler:
         
         return True
     
-    def cleanup_old_uploads(self):
+    @staticmethod
-        """Remove uploaded files older than CLEANUP_DAYS days."""
+    def _parse_upload_timestamp(value: Any) -> Optional[datetime]:
+        if not isinstance(value, str) or not value.strip():
+            return None
         try:
-            cutoff_date = datetime.now() - timedelta(days=self.cleanup_days)
+            parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
+            if parsed.tzinfo is not None:
+                parsed = parsed.astimezone().replace(tzinfo=None)
+            return parsed
+        except (TypeError, ValueError):
+            return None
+
+    @classmethod
+    def _upload_metadata_is_recent(cls, info: Dict[str, Any], cutoff_date: datetime) -> bool:
+        """Return True when upload metadata records activity inside retention."""
+        for field in ("last_accessed", "created_at", "uploaded_at"):
+            parsed = cls._parse_upload_timestamp(info.get(field))
+            if parsed is None:
+                continue
+            if parsed >= cutoff_date:
+                return True
+        return False
+
+    @classmethod
+    def _upload_index_keys_for_file(
+        cls,
+        upload_index: Dict[str, Any],
+        upload_id: str,
+        file_path: str,
+    ) -> list[str]:
+        """Find a coherent set of index rows for one physical upload.
+
+        Every related row must agree on ID, canonical path, owner, and a
+        non-empty checksum. Each row must also contain the complete lifecycle
+        timestamps written for new uploads. Ambiguous or incomplete index
+        state cannot authorize destructive cleanup.
+        """
+        target_path = os.path.normcase(os.path.realpath(file_path))
+        matches: list[str] = []
+        owners: set[str] = set()
+        checksums: set[str] = set()
+        for key, info in upload_index.items():
+            if not isinstance(info, dict):
+                continue
+            stored_path = info.get("path")
+            stored_real_path = (
+                os.path.normcase(os.path.realpath(stored_path))
+                if isinstance(stored_path, str) and stored_path
+                else None
+            )
+            same_id = info.get("id") == upload_id
+            same_path = stored_real_path == target_path
+            if not same_id and not same_path:
+                continue
+            if not same_id or not same_path:
+                logger.warning(
+                    "Skipping ambiguous cleanup candidate %s: related row has id=%r path=%r",
+                    file_path,
+                    info.get("id"),
+                    stored_path,
+                )
+                return []
+
+            owner = info.get("owner")
+            if not isinstance(owner, str) or not owner.strip():
+                logger.warning(
+                    "Skipping incomplete cleanup candidate %s: matching row has no owner",
+                    file_path,
+                )
+                return []
+
+            row_checksums = {
+                str(info.get(field)).strip().lower()
+                for field in ("hash", "checksum_sha256")
+                if info.get(field) is not None and str(info.get(field)).strip()
+            }
+            if not row_checksums:
+                logger.warning(
+                    "Skipping incomplete cleanup candidate %s: matching row has no checksum",
+                    file_path,
+                )
+                return []
+            if len(row_checksums) != 1:
+                logger.warning(
+                    "Skipping ambiguous cleanup candidate %s: matching row has conflicting checksums",
+                    file_path,
+                )
+                return []
+
+            lifecycle_fields = ("uploaded_at", "created_at", "last_accessed")
+            if any(
+                cls._parse_upload_timestamp(info.get(field)) is None
+                for field in lifecycle_fields
+            ):
+                logger.warning(
+                    "Skipping incomplete cleanup candidate %s: matching row lacks lifecycle timestamps",
+                    file_path,
+                )
+                return []
+
+            matches.append(key)
+            owners.add(owner)
+            checksums.update(row_checksums)
+
+        if len(owners) > 1 or len(checksums) > 1:
+            logger.warning(
+                "Skipping ambiguous cleanup candidate %s: matching rows disagree on owner or checksum",
+                file_path,
+            )
+            return []
+        return matches
+
+    def cleanup_old_uploads(
+        self,
+        referenced_upload_ids: Optional[set[str]] = None,
+        referenced_upload_hashes: Optional[set[str]] = None,
+    ):
+        """Remove expired uploads proven unreferenced by a complete snapshot.
+
+        ``None`` means reference discovery was not completed, so cleanup fails
+        closed and removes nothing. The admin route supplies both sets after
+        scanning persisted chats, documents, and gallery records.
+        """
+        if referenced_upload_ids is None or referenced_upload_hashes is None:
+            logger.warning("Upload cleanup skipped: persisted reference snapshot unavailable")
+            return 0
+
+        try:
+            cleanup_started_at = datetime.now()
+            cutoff_date = cleanup_started_at - timedelta(days=self.cleanup_days)
             cleaned_count = 0
-            
+
-            for root, dirs, files in os.walk(self.upload_dir):
+            referenced_ids = {str(value) for value in referenced_upload_ids}
-                if root == self.upload_dir:
+            referenced_hashes = {str(value) for value in referenced_upload_hashes}
-                    continue
+            uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
-                    
+
-                path_parts = root.split(os.sep)
+            # Keep index mutation and file removal serialized with upload writes.
-                if len(path_parts) >= 4:
+            # Each row removal is atomically persisted before the bytes are
+            # deleted; if deletion fails, the previous index is restored.
+            with self._index_lock:
+                current_index = dict(self._load_upload_index(fail_on_error=True))
+
+                for root, dirs, files in os.walk(self.upload_dir, followlinks=False):
+                    is_junction = getattr(os.path, "isjunction", lambda _path: False)
+                    dirs[:] = [
+                        directory
+                        for directory in dirs
+                        if not os.path.islink(os.path.join(root, directory))
+                        and not is_junction(os.path.join(root, directory))
+                    ]
+                    if root == self.upload_dir:
+                        continue
+                    if not self._inside_upload_dir(root):
+                        dirs[:] = []
+                        continue
+
+                    path_parts = root.split(os.sep)
+                    if len(path_parts) < 4:
+                        continue
                     try:
                         dir_date = datetime(int(path_parts[-3]), int(path_parts[-2]), int(path_parts[-1]))
-                        if dir_date < cutoff_date:
-                            for file in files:
-                                file_path = os.path.join(root, file)
-                                try:
-                                    os.remove(file_path)
-                                    cleaned_count += 1
-                                    logger.info(f"Cleaned up old upload: {file_path}")
-                                except Exception as e:
-                                    logger.warning(f"Failed to remove {file_path}: {e}")
-                            
-                            try:
-                                os.rmdir(root)
-                                logger.info(f"Removed empty upload directory: {root}")
-                            except Exception as e:
-                                logger.warning(f"Failed to remove directory {root}: {e}")
                     except (ValueError, IndexError):
                         continue
-            
+                    if dir_date >= cutoff_date:
+                        continue
+
+                    for file in files:
+                        # Reference discovery only understands canonical upload
+                        # IDs; unknown files fail closed instead of being swept.
+                        if not self.validate_upload_id(file):
+                            continue
+
+                        file_path = os.path.join(root, file)
+                        if not self._inside_upload_dir(file_path):
+                            logger.warning(
+                                "Skipping cleanup candidate outside upload directory: %s",
+                                file_path,
+                            )
+                            continue
+                        matching_keys = self._upload_index_keys_for_file(
+                            current_index,
+                            file,
+                            file_path,
+                        )
+                        matching_rows = [
+                            current_index[key]
+                            for key in matching_keys
+                            if isinstance(current_index.get(key), dict)
+                        ]
+
+                        # Files without authoritative live index rows are not
+                        # eligible for destructive cleanup. Reference hashes,
+                        # recency, and ownership cannot be proven for them.
+                        if not matching_rows:
+                            continue
+
+                        is_referenced = file in referenced_ids or any(
+                            str(info.get("id") or "") in referenced_ids
+                            or str(info.get("hash") or "") in referenced_hashes
+                            or str(info.get("checksum_sha256") or "") in referenced_hashes
+                            for info in matching_rows
+                        )
+                        metadata_is_recent = any(
+                            self._upload_metadata_is_recent(info, cutoff_date)
+                            for info in matching_rows
+                        )
+                        if is_referenced or metadata_is_recent:
+                            continue
+
+                        reduced_index = {
+                            key: value
+                            for key, value in current_index.items()
+                            if key not in matching_keys
+                        }
+                        if matching_keys:
+                            try:
+                                self._atomic_write_json(
+                                    uploads_db_path,
+                                    reduced_index,
+                                    sync_backup=True,
+                                )
+                            except Exception as e:
+                                try:
+                                    self._atomic_write_json(
+                                        uploads_db_path,
+                                        current_index,
+                                        sync_backup=True,
+                                    )
+                                except Exception:
+                                    logger.exception(
+                                        "Failed to restore upload indexes after reconciliation failed for %s",
+                                        file_path,
+                                    )
+                                    raise UploadCleanupSafetyError(
+                                        "upload index rollback failed before file removal"
+                                    ) from e
+                                logger.warning(
+                                    "Failed to reconcile upload index before removing %s: %s",
+                                    file_path,
+                                    e,
+                                )
+                                continue
+
+                        try:
+                            os.remove(file_path)
+                        except FileNotFoundError:
+                            # The bytes are already absent. Keep the reduced
+                            # lifecycle index instead of recreating a stale row.
+                            current_index = reduced_index
+                            logger.info(
+                                "Reconciled missing expired upload from index: %s",
+                                file_path,
+                            )
+                            continue
+                        except Exception as e:
+                            if matching_keys:
+                                try:
+                                    self._atomic_write_json(
+                                        uploads_db_path,
+                                        current_index,
+                                        sync_backup=True,
+                                    )
+                                except Exception:
+                                    logger.exception(
+                                        "Failed to restore upload index after removal failed for %s",
+                                        file_path,
+                                    )
+                                    raise UploadCleanupSafetyError(
+                                        "upload index rollback failed after file removal was refused"
+                                    ) from e
+                            logger.warning(f"Failed to remove {file_path}: {e}")
+                            continue
+
+                        current_index = reduced_index
+                        cleaned_count += 1
+                        logger.info(f"Cleaned up old unreferenced upload: {file_path}")
+
+                    try:
+                        if not os.listdir(root):
+                            os.rmdir(root)
+                            logger.info(f"Removed empty upload directory: {root}")
+                    except Exception as e:
+                        logger.warning(f"Failed to inspect/remove directory {root}: {e}")
+
             logger.info(f"Upload cleanup completed: {cleaned_count} files removed")
             return cleaned_count
         except Exception as e:
             logger.error(f"Upload cleanup failed: {e}")
-            return 0
+            raise UploadCleanupSafetyError("upload cleanup safety checks failed") from e
     
     def validate_upload_id(self, upload_id: str) -> bool:
         """Validate that the upload ID matches the expected pattern."""
@@ -293,71 +672,103 @@ class UploadHandler:
 
     def _inside_upload_dir(self, path: str) -> bool:
         """Check if path is inside the upload directory."""
-        base = os.path.realpath(self.upload_dir)
+        base = os.path.normcase(os.path.realpath(self.upload_dir))
-        p = os.path.realpath(path)
+        p = os.path.normcase(os.path.realpath(path))
         try:
             return os.path.commonpath([base, p]) == base
         except Exception:
             return False
 
-    def _atomic_write_json(self, path: str, data: dict) -> None:
+    def _atomic_write_json(
+        self,
+        path: str,
+        data: dict,
+        *,
+        sync_backup: bool = False,
+    ) -> None:
         """Write `data` to `path` atomically: write to a temp file in the
         same directory, then `os.replace` onto the target. The kernel
         guarantees `os.replace` is atomic on POSIX, so a reader either
         sees the old contents or the new contents, never a half-written
-        file. Also keeps a `.bak` sibling of the previous good state.
+        file. Normally `.bak` retains the previous good state. Destructive
+        lifecycle transitions use ``sync_backup=True`` so recovery cannot
+        resurrect metadata for bytes that were deliberately removed.
         """
         directory = os.path.dirname(path) or "."
-        fd, tmp = tempfile.mkstemp(prefix=".uploads-", suffix=".tmp", dir=directory)
+
-        try:
+        def _replace_json(target: str) -> None:
-            with os.fdopen(fd, "w", encoding="utf-8") as f:
+            fd, tmp = tempfile.mkstemp(
-                json.dump(data, f, indent=2)
+                prefix=".uploads-",
-                f.flush()
+                suffix=".tmp",
-                os.fsync(f.fileno())
+                dir=directory,
-            if os.path.exists(path):
+            )
-                bak = path + ".bak"
+            try:
+                with os.fdopen(fd, "w", encoding="utf-8") as f:
+                    json.dump(data, f, indent=2)
+                    f.flush()
+                    os.fsync(f.fileno())
+                os.replace(tmp, target)
+            except Exception:
                 try:
-                    shutil.copy2(path, bak)
+                    os.unlink(tmp)
                 except OSError:
                     pass
-            os.replace(tmp, path)
+                raise
-            # Update cache if this is the main index
+
-            if path.endswith("uploads.json"):
+        if sync_backup:
-                self._index_cache = data
+            _replace_json(path + ".bak")
-                try:
+        elif os.path.exists(path):
-                    self._index_mtime = os.path.getmtime(path)
-                except OSError:
-                    self._index_mtime = time.time()
-        except Exception:
             try:
-                os.unlink(tmp)
+                shutil.copy2(path, path + ".bak")
             except OSError:
                 pass
-            raise
 
-    def _load_upload_index(self) -> Dict[str, Any]:
+        _replace_json(path)
+        # Update cache if this is the main index
+        if path.endswith("uploads.json"):
+            self._index_cache = data
+            try:
+                self._index_mtime = os.path.getmtime(path)
+            except OSError:
+                self._index_mtime = time.time()
+
+    def _load_upload_index(self, *, fail_on_error: bool = False) -> Dict[str, Any]:
         """Load the upload index from disk/cache. Uses mtime-based validation
-        to avoid redundant parsing on hot paths.
+        to avoid redundant parsing on hot paths. When ``fail_on_error`` is
+        true, a missing, malformed, or unreadable live index raises so
+        destructive callers cannot mistake corruption for an empty store.
         """
         uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
-        if not os.path.exists(uploads_db_path):
+        candidates = (uploads_db_path, uploads_db_path + ".bak")
+        if fail_on_error:
+            # A backup is intentionally the previous snapshot. It is useful for
+            # non-destructive reads, but cannot authorize deletion when the live
+            # index is missing or corrupt.
+            if not os.path.exists(uploads_db_path):
+                raise ValueError("live uploads database is missing")
+            existing_candidates = [uploads_db_path]
+        else:
+            existing_candidates = [path for path in candidates if os.path.exists(path)]
+        if not existing_candidates:
             self._index_cache = {}
             self._index_mtime = 0.0
             return {}
 
         # Check cache validity
         try:
-            mtime = os.path.getmtime(uploads_db_path)
+            mtime = max(os.path.getmtime(path) for path in existing_candidates)
-            if self._index_cache is not None and mtime <= self._index_mtime:
+            if (
+                not fail_on_error
+                and self._index_cache is not None
+                and mtime <= self._index_mtime
+            ):
                 return self._index_cache
         except OSError:
             mtime = 0.0
 
         # Try the live file first, fall back to the .bak sibling if the
         # live file is truncated/corrupted.
-        for candidate in (uploads_db_path, uploads_db_path + ".bak"):
+        for candidate in existing_candidates:
-            if not os.path.exists(candidate):
-                continue
             try:
                 with open(candidate, "r", encoding="utf-8") as f:
                     data = json.load(f)
@@ -369,6 +780,8 @@ class UploadHandler:
                 logger.warning(f"Failed to read uploads database ({candidate}): {e}")
                 continue
 
+        if fail_on_error:
+            raise ValueError("live uploads database is unreadable")
         self._index_cache = {}
         return {}
 
@@ -381,6 +794,144 @@ class UploadHandler:
                 return dict(info)
         return None
 
+    def reserve_upload(
+        self,
+        upload_id: str,
+        *,
+        owner: Optional[str],
+        auth_manager: Any = None,
+        allow_admin: bool = False,
+    ) -> Optional[Dict[str, Any]]:
+        """Owner-check and reserve an indexed upload against cleanup.
+
+        The live index lookup, ownership/path validation, and access touch all
+        occur under the cleanup lock. A durable-reference writer must not
+        commit when this returns ``None``.
+        """
+        if not self.validate_upload_id(upload_id):
+            return None
+
+        auth_configured = bool(auth_manager and getattr(auth_manager, "is_configured", False))
+        if auth_configured and not owner:
+            return None
+
+        uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
+        with self._index_lock:
+            try:
+                current = dict(self._load_upload_index(fail_on_error=True))
+            except Exception:
+                logger.warning("Cannot reserve upload %s without a valid live index", upload_id)
+                return None
+            matching_keys = [
+                key
+                for key, info in current.items()
+                if isinstance(info, dict) and info.get("id") == upload_id
+            ]
+            if not matching_keys:
+                return None
+
+            matching_rows = [dict(current[key]) for key in matching_keys]
+            row_owners = {
+                str(row.get("owner")) if row.get("owner") is not None else None
+                for row in matching_rows
+            }
+            row_hashes = {
+                str(row.get("hash") or row.get("checksum_sha256"))
+                for row in matching_rows
+                if row.get("hash") or row.get("checksum_sha256")
+            }
+            if len(row_owners) != 1 or len(row_hashes) > 1:
+                logger.warning(
+                    "Cannot reserve ambiguous upload index rows for %s",
+                    upload_id,
+                )
+                return None
+
+            is_admin = False
+            if allow_admin and owner and auth_manager and hasattr(auth_manager, "is_admin"):
+                try:
+                    is_admin = bool(auth_manager.is_admin(owner))
+                except Exception:
+                    is_admin = False
+
+            now = datetime.now()
+            current_info = matching_rows[0]
+            if owner and not is_admin and current_info.get("owner") != owner:
+                return None
+            if not owner and current_info.get("owner") is not None:
+                return None
+
+            existing_paths: set[str] = set()
+            for row in matching_rows:
+                stored_path = row.get("path")
+                if not stored_path:
+                    continue
+                if not self._inside_upload_dir(stored_path):
+                    logger.warning(
+                        "Cannot reserve upload %s with an out-of-root index path",
+                        upload_id,
+                    )
+                    return None
+                if os.path.isfile(stored_path):
+                    if os.path.basename(stored_path) != upload_id:
+                        return None
+                    existing_paths.add(os.path.normcase(os.path.realpath(stored_path)))
+            if len(existing_paths) > 1:
+                logger.warning("Cannot reserve upload %s with multiple indexed paths", upload_id)
+                return None
+            path = next(iter(existing_paths), None) or self._find_upload_path(upload_id)
+            if not path or not os.path.isfile(path) or not self._inside_upload_dir(path):
+                return None
+
+            last_accessed = self._parse_upload_timestamp(current_info.get("last_accessed"))
+            path_changed = current_info.get("path") != path
+            needs_write = (
+                path_changed
+                or last_accessed is None
+                or last_accessed < now - timedelta(minutes=5)
+            )
+            if needs_write:
+                accessed_at = now.isoformat()
+                updated_index = dict(current)
+                for key in matching_keys:
+                    updated = dict(updated_index[key])
+                    updated["path"] = path
+                    updated["last_accessed"] = accessed_at
+                    updated_index[key] = updated
+                try:
+                    self._atomic_write_json(
+                        uploads_db_path,
+                        updated_index,
+                        sync_backup=True,
+                    )
+                except Exception:
+                    try:
+                        self._atomic_write_json(
+                            uploads_db_path,
+                            current,
+                            sync_backup=True,
+                        )
+                    except Exception:
+                        logger.exception(
+                            "Failed to restore upload indexes after reservation failed for %s",
+                            upload_id,
+                        )
+                    logger.exception("Failed to reserve upload %s against cleanup", upload_id)
+                    return None
+                current_info = dict(updated_index[matching_keys[0]])
+
+            resolved = dict(current_info)
+            resolved.setdefault("id", upload_id)
+            resolved["path"] = path
+            resolved.setdefault("name", os.path.basename(path))
+            resolved.setdefault("original_name", resolved["name"])
+            resolved.setdefault("mime", mimetypes.guess_type(path)[0] or "application/octet-stream")
+            if resolved.get("hash") and not resolved.get("checksum_sha256"):
+                resolved["checksum_sha256"] = resolved["hash"]
+            if resolved.get("uploaded_at") and not resolved.get("created_at"):
+                resolved["created_at"] = resolved["uploaded_at"]
+            return resolved
+
     def _renamed_upload_index_key(self, key: str, info: Dict[str, Any], old_owner: str, new_owner: str) -> str:
         """Return the storage key to use after renaming an owned upload row.
 
@@ -475,16 +1026,32 @@ class UploadHandler:
         if not self.validate_upload_id(upload_id):
             return None
 
+        candidates: list[str] = []
         direct = os.path.join(self.upload_dir, upload_id)
-        if os.path.exists(direct) and self._inside_upload_dir(direct):
+        if os.path.isfile(direct) and self._inside_upload_dir(direct):
-            return direct
+            candidates.append(os.path.realpath(direct))
 
-        for root, _dirs, files in os.walk(self.upload_dir, followlinks=False):
+        for root, dirs, files in os.walk(self.upload_dir, followlinks=False):
+            is_junction = getattr(os.path, "isjunction", lambda _path: False)
+            dirs[:] = [
+                directory
+                for directory in dirs
+                if not os.path.islink(os.path.join(root, directory))
+                and not is_junction(os.path.join(root, directory))
+            ]
             if upload_id in files:
                 path = os.path.join(root, upload_id)
-                if self._inside_upload_dir(path):
+                if os.path.isfile(path) and self._inside_upload_dir(path):
-                    return path
+                    real_path = os.path.realpath(path)
-        return None
+                    if real_path not in candidates:
+                        candidates.append(real_path)
+                        if len(candidates) > 1:
+                            logger.warning(
+                                "Upload ID %s resolves to multiple physical files",
+                                upload_id,
+                            )
+                            return None
+        return candidates[0] if candidates else None
 
     def resolve_upload(
         self,
@@ -493,52 +1060,20 @@ class UploadHandler:
         auth_manager: Any = None,
         allow_admin: bool = True,
     ) -> Optional[Dict[str, Any]]:
-        """Resolve an upload ID to metadata only if the caller may read it.
+        """Resolve and reserve an upload only if the caller may read it.
 
         This is the owner-aware lookup used by internal processors. Public
         download routes already perform owner checks; chat/document paths must
-        do the same before reading file bytes server-side.
+        do the same before reading file bytes server-side. Reservation shares
+        cleanup's lifecycle lock and prevents a newly persisted reference from
+        racing final deletion.
         """
-        if not self.validate_upload_id(upload_id):
+        return self.reserve_upload(
-            logger.warning(f"Invalid upload ID format: {upload_id}")
+            upload_id,
-            return None
+            owner=owner,
-
+            auth_manager=auth_manager,
-        auth_configured = bool(auth_manager and getattr(auth_manager, "is_configured", False))
+            allow_admin=allow_admin,
-        if auth_configured and not owner:
+        )
-            return None
-
-        info = self.get_upload_info(upload_id) or {}
-        is_admin = False
-        if allow_admin and owner and auth_manager and hasattr(auth_manager, "is_admin"):
-            try:
-                is_admin = bool(auth_manager.is_admin(owner))
-            except Exception:
-                is_admin = False
-
-        if owner and not is_admin:
-            if info.get("owner") != owner:
-                logger.warning("Upload %s denied for owner %s", upload_id, owner)
-                return None
-        if not owner and info.get("owner") is not None:
-            logger.warning("Upload %s denied without an authenticated owner", upload_id)
-            return None
-
-        path = info.get("path")
-        if not path or not os.path.exists(path) or not self._inside_upload_dir(path):
-            path = self._find_upload_path(upload_id)
-        if not path:
-            return None
-        if not self._inside_upload_dir(path):
-            logger.warning(f"Upload path outside upload directory: {path}")
-            return None
-
-        resolved = dict(info)
-        resolved.setdefault("id", upload_id)
-        resolved["path"] = path
-        resolved.setdefault("name", os.path.basename(path))
-        resolved.setdefault("original_name", resolved["name"])
-        resolved.setdefault("mime", mimetypes.guess_type(path)[0] or "application/octet-stream")
-        return resolved
     
     def cleanup_rate_limits(self):
         """Remove stale entries from upload_rate_log."""
@@ -706,6 +1241,9 @@ class UploadHandler:
                         # fresh-insert path below; release the lock first.
                         raise LookupError("upload entry vanished mid-dedupe")
                     existing_file["last_accessed"] = datetime.now().isoformat()
+                    existing_file.setdefault("checksum_sha256", file_hash)
+                    if existing_file.get("uploaded_at"):
+                        existing_file.setdefault("created_at", existing_file["uploaded_at"])
                     current[live_key] = existing_file
                     self._atomic_write_json(uploads_db_path, current)
                 except LookupError:
@@ -721,7 +1259,9 @@ class UploadHandler:
                     "size": existing_file["size"],
                     "name": existing_file["original_name"],
                     "hash": file_hash,
+                    "checksum_sha256": existing_file.get("checksum_sha256") or file_hash,
                     "uploaded_at": existing_file["uploaded_at"],
+                    "created_at": existing_file.get("created_at") or existing_file["uploaded_at"],
                     "owner": existing_file.get("owner"),
                     "width": existing_file.get("width"),
                     "height": existing_file.get("height"),
@@ -744,6 +1284,7 @@ class UploadHandler:
             raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
         
         # Create file metadata
+        created_at = datetime.now().isoformat()
         file_metadata = {
             "id": file_id,
             "path": file_path,
@@ -751,9 +1292,11 @@ class UploadHandler:
             "size": file_size,
             "name": safe_filename,
             "hash": file_hash,
+            "checksum_sha256": file_hash,
             "original_name": original_filename,
-            "uploaded_at": datetime.now().isoformat(),
+            "uploaded_at": created_at,
-            "last_accessed": datetime.now().isoformat(),
+            "created_at": created_at,
+            "last_accessed": created_at,
             "client_ip": client_ip,
             "owner": owner,
         }

src/webhook_manager.py

@@ -7,10 +7,12 @@ import ipaddress
 import json
 import logging
 import re
+import ssl
 from datetime import datetime, timezone
 from typing import Optional
 from urllib.parse import urlparse
 
+import httpcore
 import httpx
 
 from src.database import SessionLocal, Webhook
@@ -125,6 +127,128 @@ def validate_webhook_url(url: str) -> str:
     return url
 
 
+def _validated_public_ips(url: str) -> list:
+    """Resolve *url*'s host and return its IPs, raising ValueError if any is
+    private/internal.
+
+    ``validate_webhook_url`` resolves the host to decide accept/reject, but the
+    subsequent ``httpx`` connect re-resolves independently — so a DNS record
+    that flips between the two lookups (rebinding) can slip an internal IP past
+    the check. Callers pin the delivery connection to the IP this function
+    returns, closing that TOCTOU. Fail closed: an unresolvable or partly-private
+    result raises rather than returning a usable IP.
+    """
+    parsed = urlparse(url)
+    hostname = (parsed.hostname or "").strip()
+    if not hostname:
+        raise ValueError("URL must have a hostname")
+    try:
+        literal = ipaddress.ip_address(hostname)
+    except ValueError:
+        literal = None
+    if literal is not None:
+        if _ip_is_private(literal):
+            raise ValueError("URL must not point to private/internal addresses")
+        return [literal]
+    addrs = _resolve_hostname_ips(hostname)
+    if not addrs or any(_ip_is_private(a) for a in addrs):
+        raise ValueError("URL must not point to private/internal addresses")
+    return addrs
+
+
+# httpcore raises its own exception hierarchy; map the ones a simple POST can
+# surface back to their httpx equivalents so callers' `except httpx.*` blocks
+# (and sanitize_error) behave exactly as they did with the default transport.
+_HTTPCORE_TO_HTTPX_EXC = {
+    httpcore.ConnectError: httpx.ConnectError,
+    httpcore.ConnectTimeout: httpx.ConnectTimeout,
+    httpcore.NetworkError: httpx.NetworkError,
+    httpcore.PoolTimeout: httpx.PoolTimeout,
+    httpcore.ProtocolError: httpx.ProtocolError,
+    httpcore.ReadError: httpx.ReadError,
+    httpcore.ReadTimeout: httpx.ReadTimeout,
+    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
+    httpcore.TimeoutException: httpx.TimeoutException,
+    httpcore.WriteError: httpx.WriteError,
+    httpcore.WriteTimeout: httpx.WriteTimeout,
+}
+
+
+class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend):
+    """Async network backend that routes every TCP connect to a fixed IP.
+
+    httpcore derives TLS SNI and the ``Host`` header from the request URL, not
+    from the connect host, so pinning only the socket destination keeps
+    certificate validation and vhost routing pointed at the original hostname.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress):
+        self._ip = str(ip)
+        self._real = httpcore.AnyIOBackend()
+
+    async def connect_tcp(self, host, port, timeout=None, local_address=None,
+                          socket_options=None):
+        return await self._real.connect_tcp(
+            self._ip, port, timeout, local_address, socket_options
+        )
+
+    async def connect_unix_socket(self, path, timeout=None, socket_options=None):
+        return await self._real.connect_unix_socket(path, timeout, socket_options)
+
+    async def sleep(self, seconds: float) -> None:
+        return await self._real.sleep(seconds)
+
+
+class _PinnedAsyncTransport(httpx.AsyncBaseTransport):
+    """httpx transport that pins the TCP connect to a pre-resolved public IP.
+
+    Uses only public ``httpcore`` / ``httpx`` APIs. The request URL is passed
+    through unchanged (Host + SNI stay the original hostname); only the socket
+    destination is pinned, closing the DNS-rebinding TOCTOU between the SSRF
+    check and the connect. HTTP/1.1 only — webhook deliveries are small POSTs.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress):
+        self._pool = httpcore.AsyncConnectionPool(
+            ssl_context=ssl.create_default_context(),
+            http1=True,
+            http2=False,
+            network_backend=_PinnedAsyncBackend(ip),
+        )
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        core_req = httpcore.Request(
+            method=request.method,
+            url=httpcore.URL(
+                scheme=request.url.raw_scheme,
+                host=request.url.raw_host,
+                port=request.url.port,
+                target=request.url.raw_path,
+            ),
+            headers=request.headers.raw,
+            content=request.stream,
+            extensions=request.extensions,
+        )
+        try:
+            core_resp = await self._pool.handle_async_request(core_req)
+            content = b"".join([chunk async for chunk in core_resp.aiter_stream()])
+            await core_resp.aclose()
+        except Exception as exc:
+            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
+            if mapped is not None:
+                raise mapped(str(exc)) from exc
+            raise
+        return httpx.Response(
+            status_code=core_resp.status,
+            headers=core_resp.headers,
+            content=content,
+            extensions=core_resp.extensions,
+        )
+
+    async def aclose(self) -> None:
+        await self._pool.aclose()
+
+
 def validate_events(events_str: str) -> str:
     """Validate comma-separated event names. Returns cleaned string."""
     events = [e.strip() for e in events_str.split(",") if e.strip()]
@@ -198,8 +322,11 @@ def sanitize_error(error: str, max_len: int = 200) -> str:
 
 class WebhookManager:
     def __init__(self, api_key_manager=None):
-        # Disable redirects to prevent SSRF via redirect chains
+        # No shared client: each delivery builds a short-lived client whose
-        self._client = httpx.AsyncClient(timeout=10, follow_redirects=False)
+        # transport is pinned to the SSRF-approved IP (see _deliver /
+        # _send_request), so a single reusable client can't be pointed at
+        # different pinned hosts. Redirects stay disabled on every delivery
+        # client to prevent SSRF via redirect chains.
         self._loop: Optional[asyncio.AbstractEventLoop] = None
         self._api_key_manager = api_key_manager
         # Strong references to in-flight fire-and-forget tasks. asyncio only
@@ -262,11 +389,28 @@ class WebhookManager:
         decrypted = self._decrypt_secret(encrypted_secret)
         await self._deliver(webhook_id, url, decrypted, "webhook.test", {"message": "Test ping from Odysseus"})
 
+    async def _send_request(self, url: str, body: str, headers: dict,
+                            ip: ipaddress._BaseAddress) -> httpx.Response:
+        """POST *body* to *url* with the TCP connect pinned to *ip*.
+
+        Overridable seam: tests replace this to avoid real sockets. Redirects
+        are disabled so a 3xx can't bounce the delivery to another host.
+        """
+        transport = _PinnedAsyncTransport(ip)
+        async with httpx.AsyncClient(
+            timeout=10, follow_redirects=False, transport=transport,
+        ) as client:
+            return await client.post(url, content=body, headers=headers)
+
     async def _deliver(self, webhook_id: str, url: str, secret: Optional[str], event: str, payload: dict):
         """Internal delivery. Never call directly from outside this class (use deliver_test)."""
-        # Re-validate URL at delivery time in case DB was tampered with
+        # Re-validate URL at delivery time in case DB was tampered with, and
+        # capture the exact IPs that passed the check so the connect can be
+        # pinned to one of them (closes the DNS-rebinding TOCTOU: the check
+        # below and the socket connect no longer resolve independently).
         try:
             validate_webhook_url(url)
+            pinned_ips = _validated_public_ips(url)
         except ValueError as e:
             logger.warning(f"Webhook {webhook_id} has invalid URL, skipping: {e}")
             return
@@ -283,7 +427,7 @@ class WebhookManager:
 
         db = SessionLocal()
         try:
-            resp = await self._client.post(url, content=body, headers=headers)
+            resp = await self._send_request(url, body, headers, pinned_ips[0])
             db.query(Webhook).filter(Webhook.id == webhook_id).update({
                 "last_triggered_at": _utcnow(),
                 "last_status_code": resp.status_code,
@@ -305,4 +449,7 @@ class WebhookManager:
             db.close()
 
     async def close(self):
-        await self._client.aclose()
+        # Delivery clients are per-request and closed via their async context
+        # manager, so there is no long-lived client to tear down here. Kept for
+        # API compatibility with callers (e.g. app shutdown).
+        return None

static/app.js

@@ -53,6 +53,73 @@ window.uiModule = uiModule;
 window.adminModule = adminModule;
 window.cookbookModule = cookbookModule;
 
+function _isMobileChatInput() {
+  return window.innerWidth <= 768;
+}
+
+function _submitChatFormDirect(form) {
+  if (!form) return;
+  if (form.requestSubmit) form.requestSubmit();
+  else form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
+}
+
+function _isForegroundChatBusy() {
+  const sendBtn = document.querySelector('.send-btn');
+  return !!window.__odysseusChatBusy
+    || Date.now() < (window.__odysseusChatBusyUntil || 0)
+    || !!document.querySelector('.send-btn[data-mode="streaming"], .send-btn.send-pending')
+    || (sendBtn && (sendBtn.title || '').toLowerCase().includes('stop'));
+}
+
+function _shouldQueueFromMobileEnter(e, input) {
+  return e.key === 'Enter'
+    && !e.shiftKey
+    && !e.ctrlKey
+    && !e.metaKey
+    && !e.altKey
+    && !e.isComposing
+    && _isMobileChatInput()
+    && _isForegroundChatBusy()
+    && !!(input && input.value && input.value.trim());
+}
+
+function _shouldQueueFromMobileLineBreak(input) {
+  return _isMobileChatInput()
+    && _isForegroundChatBusy()
+    && !!(input && input.value && input.value.trim());
+}
+
+function _isLineBreakInputEvent(e) {
+  return e
+    && (e.inputType === 'insertLineBreak'
+      || e.inputType === 'insertParagraph'
+      || e.data === '\n');
+}
+
+function _submitMobileQueuedInput(input) {
+  if (!input || !_shouldQueueFromMobileLineBreak(input)) return false;
+  const now = Date.now();
+  const last = Number(input.dataset.mobileQueueSubmitAt || 0);
+  if (now - last < 300) return true;
+  input.dataset.mobileQueueSubmitAt = String(now);
+  if (chatModule && chatModule.queueStreamingComposerRequest && chatModule.queueStreamingComposerRequest()) {
+    return true;
+  }
+  window.__odysseusQueueStreamingSubmit = now;
+  const form = document.getElementById('chat-form');
+  _submitChatFormDirect(form);
+  return true;
+}
+
+function _syncMobileEnterKeyHint(input) {
+  if (!input) return;
+  input.setAttribute('enterkeyhint', (_isMobileChatInput() && _isForegroundChatBusy()) ? 'send' : 'enter');
+}
+
+function _countLineBreaks(s) {
+  return ((s || '').match(/\n/g) || []).length;
+}
+
 function initForegroundActivityHeartbeat() {
   let lastSent = 0;
   const minGapMs = 12000;
@@ -185,6 +252,50 @@ async function _createDirectChatFromPreferredModel() {
   return false;
 }
 
+async function _hasUsableChatModel() {
+  try {
+    const pending = sessionModule?.getPendingChat?.();
+    if (pending && pending.url && pending.modelId) return true;
+  } catch (_) {}
+  try {
+    const current = sessionModule?.getSessions?.()
+      ?.find(s => s.id === sessionModule?.getCurrentSessionId?.());
+    if (current && current.endpoint_url && current.model) return true;
+  } catch (_) {}
+  const dc = await _refreshDefaultChat();
+  if (dc && dc.endpoint_url && dc.model) return true;
+  try {
+    const items = window.modelsModule?.getCachedItems?.() || [];
+    if (items.some(item => !item.offline && ((item.models || []).length || (item.models_extra || []).length))) {
+      return true;
+    }
+  } catch (_) {}
+  try {
+    const res = await fetch(`${API_BASE}/api/models?background=false`, { credentials: 'same-origin' });
+    if (!res.ok) return false;
+    const data = await res.json();
+    return (data.items || []).some(item => !item.offline && ((item.models || []).length || (item.models_extra || []).length));
+  } catch (_) {
+    return false;
+  }
+}
+
+async function _syncWelcomeModelHint() {
+  const tip = document.getElementById('welcome-tip');
+  const sub = document.getElementById('welcome-sub');
+  if (!tip && !sub) return;
+  const hasModel = await _hasUsableChatModel();
+  if (hasModel) {
+    if (sub && !sub.dataset.researchOrigText) sub.textContent = 'New chat ready.';
+    if (tip) tip.textContent = 'Pick a model if you want, or just type.';
+  } else {
+    if (sub && !sub.dataset.researchOrigText) {
+      sub.innerHTML = 'Welcome, <span class="setup-trigger-link" style="color:var(--accent,var(--red));font-weight:600;cursor:pointer;text-decoration:underline;" title="Click to launch setup">type /setup</span> to get started.';
+    }
+    if (tip) tip.textContent = 'Add an AI endpoint from Settings in the sidebar, or paste an endpoint/API key into the chat.';
+  }
+}
+
 // ============================================
 // EVENT LISTENERS INITIALIZATION
 // ============================================
@@ -195,9 +306,9 @@ function initializeEventListeners() {
   // File attachments (inside overflow menu)
   const _overflowAttach = el('overflow-attach-btn');
   if (_overflowAttach) _overflowAttach.addEventListener('click', fileHandlerModule.openPicker);
-  el('file-input').addEventListener('change', (e)=>{
+  el('file-input').addEventListener('change', async (e)=>{
-    for (const f of e.target.files) fileHandlerModule.addFiles([f]);
+    await fileHandlerModule.addFiles(Array.from(e.target.files || []));
-    fileHandlerModule.renderAttachStrip();
+    e.target.value = '';
     // Refocus textarea after file picker closes (mobile keyboard)
     const ta = el('message');
     if (ta) setTimeout(() => ta.focus(), 100);
@@ -211,7 +322,7 @@ function initializeEventListeners() {
       if (item.kind === 'file'){
         const f = item.getAsFile();
         if (f) {
-          fileHandlerModule.addFiles([f]);
+          await fileHandlerModule.addFiles([f]);
           changed = true;
         }
       }
@@ -371,8 +482,10 @@ function initializeEventListeners() {
         }
         const body = child.querySelector('.body');
         // Prefer dataset.raw (original markdown) over innerText (rendered HTML as text)
-        // to avoid extra newlines and formatting artifacts.
+        // to avoid extra newlines and formatting artifacts. Raw text lives on
-        const text = body ? (body.dataset.raw || body.innerText || body.textContent || '').trim() : '';
+        // the outer .msg in the main renderer; keep body.dataset.raw as a legacy
+        // fallback for older/reused render paths.
+        const text = (child.dataset?.raw || body?.dataset?.raw || body?.innerText || body?.textContent || '').trim();
         if (text) parts.push(`${label}: ${text}`);
       } else if (child.classList?.contains('agent-thread')) {
         const lines = ['[Tool calls]'];
@@ -3113,10 +3226,7 @@ function initializeEventListeners() {
     }, { passive: true });
   })();
 
-  // New session button on icon rail
+  async function _handleNewChatAction({ preferModel = true, focus = true } = {}) {
-  const railNewSession = el('rail-new-session');
-  if (railNewSession) {
-    railNewSession.addEventListener('click', async () => {
       if (!sessionModule) return;
       if (_closeCompareIfActive()) return;
       _deactivateIncognito();
@@ -3125,18 +3235,24 @@ function initializeEventListeners() {
       // Clear research mode if active
       const _resChk = el('research-toggle');
       if (_resChk && _resChk.checked) _syncResearchIndicator(false);
-      if (await _createDirectChatFromPreferredModel()) return;
+      if (preferModel && await _createDirectChatFromPreferredModel()) return;
       // No models at all — show welcome screen
-      sessionModule.setCurrentSessionId(null);
+      _startFreshChat();
-      if (documentModule && documentModule.isPanelOpen && documentModule.isPanelOpen()) documentModule.closePanel();
       const docBtn3 = el('overflow-doc-btn');
       if (docBtn3) docBtn3.classList.remove('active', 'has-docs');
-      const box = el('chat-history');
-      if (box) box.innerHTML = '';
-      if (chatModule && chatModule.showWelcomeScreen) {
-        chatModule.showWelcomeScreen();
-      }
       document.querySelectorAll('.session-item.active').forEach(s => s.classList.remove('active'));
+      if (focus) {
+        const input = el('message');
+        if (input) { try { input.focus(); } catch (_) {} }
+      }
+  }
+
+  // New session button on icon rail
+  const railNewSession = el('rail-new-session');
+  if (railNewSession) {
+    railNewSession.addEventListener('click', async (e) => {
+      if (e) { e.preventDefault(); e.stopPropagation(); }
+      await _handleNewChatAction();
     });
   }
 
@@ -3163,31 +3279,17 @@ function initializeEventListeners() {
   // Logo click → new chat (same logic as rail new-session button)
   const brandBtn = el('sidebar-brand-btn');
   if (brandBtn) {
-    brandBtn.addEventListener('click', async () => {
+    brandBtn.addEventListener('click', async (e) => {
-      if (!sessionModule) return;
+      if (e) { e.preventDefault(); e.stopPropagation(); }
-      if (_closeCompareIfActive()) return;
+      await _handleNewChatAction();
-      _deactivateIncognito();
-      if (presetsModule && presetsModule.deactivateCharacter) presetsModule.deactivateCharacter();
-      // Clear research toggle when starting a fresh chat (not via research button)
-      _syncResearchIndicator(false);
-      if (await _createDirectChatFromPreferredModel()) return;
-      // No models at all — show welcome screen
-      sessionModule.setCurrentSessionId(null);
-      if (documentModule && documentModule.isPanelOpen && documentModule.isPanelOpen()) documentModule.closePanel();
-      const docBtn2 = el('overflow-doc-btn');
-      if (docBtn2) docBtn2.classList.remove('active', 'has-docs');
-      const box = el('chat-history');
-      if (box) box.innerHTML = '';
-      if (chatModule && chatModule.showWelcomeScreen) chatModule.showWelcomeScreen();
-      document.querySelectorAll('.session-item.active').forEach(s => s.classList.remove('active'));
     });
   }
 
   const sidebarNewChatBtn = el('sidebar-new-chat-btn');
   if (sidebarNewChatBtn) {
-    sidebarNewChatBtn.addEventListener('click', () => {
+    sidebarNewChatBtn.addEventListener('click', async (e) => {
-      const brandBtn = el('sidebar-brand-btn');
+      if (e) { e.preventDefault(); e.stopImmediatePropagation(); }
-      if (brandBtn) brandBtn.click();
+      await _handleNewChatAction();
     });
   }
 
@@ -3226,17 +3328,38 @@ function initializeEventListeners() {
   // Textarea auto-resize
   const textarea = el('message');
   if (textarea) {
+    _syncMobileEnterKeyHint(textarea);
+    window.addEventListener('odysseus:chat-busy-change', () => _syncMobileEnterKeyHint(textarea));
     uiModule.autoResize(textarea);
-    textarea.addEventListener('input', () => {
+    let previousTextareaValue = textarea.value || '';
+    textarea.addEventListener('beforeinput', (e) => {
+      if (_isLineBreakInputEvent(e) && _shouldQueueFromMobileLineBreak(textarea)) {
+        e.preventDefault();
+        e.stopPropagation();
+        _submitMobileQueuedInput(textarea);
+      }
+    });
+    textarea.addEventListener('input', (e) => {
+      const currentValue = textarea.value || '';
+      const insertedLineBreak = _isLineBreakInputEvent(e)
+        || _countLineBreaks(currentValue) > _countLineBreaks(previousTextareaValue);
+      if (insertedLineBreak && _shouldQueueFromMobileLineBreak(textarea)) {
+        textarea.value = currentValue.replace(/\n+$/g, '');
+        previousTextareaValue = textarea.value || '';
+        _submitMobileQueuedInput(textarea);
+        return;
+      }
+      previousTextareaValue = currentValue;
       uiModule.autoResize(textarea);
+      _syncMobileEnterKeyHint(textarea);
     });
     textarea.addEventListener('paste', () => {
       setTimeout(() => uiModule.autoResize(textarea), 1);
     });
     textarea.addEventListener('keydown', (e) => {
-      const isMobile = window.innerWidth <= 768
+      const isMobile = _isMobileChatInput();
 
-      if (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile) {
+      if (_shouldQueueFromMobileEnter(e, textarea) || (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile)) {
         // If ghost autocomplete is active, accept the suggestion instead of submitting
         if (window._ghostAutocomplete && window._ghostAutocomplete.isActive()) {
           e.preventDefault();
@@ -3249,8 +3372,13 @@ function initializeEventListeners() {
         // Check if already submitting before triggering form submission
         const form = el('chat-form');
         if (form) {
-         const submitBtn = form.querySelector('button[type="submit"]');
+          if (_isForegroundChatBusy() && textarea.value && textarea.value.trim()) {
-         if (submitBtn) submitBtn.click();
+            if (chatModule && chatModule.queueStreamingComposerRequest && chatModule.queueStreamingComposerRequest()) {
+              return;
+            }
+            window.__odysseusQueueStreamingSubmit = Date.now();
+          }
+          _submitChatFormDirect(form);
         }
       }
     });
@@ -3429,7 +3557,7 @@ function initializeEventListeners() {
         // Now submit the form (the /new command handler will process it)
         setTimeout(() => {
           const form = el('chat-form');
-          if (form) form.querySelector('button[type="submit"]').click();
+          _submitChatFormDirect(form);
         }, 0);
       }
     };
@@ -3685,10 +3813,31 @@ function startOdysseusApp() {
     return fileHandlerModule.getPendingCount && fileHandlerModule.getPendingCount() > 0;
   }
 
+  function _updateStreamingSubmitButton() {
+    if (!sendBtn || sendBtn.dataset.mode !== 'streaming') return false;
+    const hasText = messageInput && messageInput.value.trim().length > 0;
+    const nextPhase = hasText ? 'queue' : 'processing';
+    if (sendBtn.dataset.phase === nextPhase) return true;
+    sendBtn.dataset.phase = nextPhase;
+    sendBtn.classList.remove('mic-mode', 'newchat-mode', 'newchat-expanded', 'anim-spin', 'anim-launch', 'anim-land');
+    if (hasText) {
+      sendBtn.innerHTML = _sendIcon;
+      sendBtn.title = 'Queue message';
+    } else {
+      sendBtn.innerHTML = _stopIcon;
+      sendBtn.title = 'Stop generation';
+    }
+    return true;
+  }
+
   function _updateSendBtnIcon() {
     if (!sendBtn) return;
-    // Don't override if streaming (stop button) or recording
+    if (sendBtn.dataset.mode === 'streaming') {
-    if (sendBtn.dataset.mode === 'streaming' || sendBtn.dataset.mode === 'recording') return;
+      _updateStreamingSubmitButton();
+      return;
+    }
+    // Don't override if recording
+    if (sendBtn.dataset.mode === 'recording') return;
     const prevMode = sendBtn.dataset.mode || '';
     const hasText = messageInput && messageInput.value.trim().length > 0;
     const hasFiles = _hasAttachments();
@@ -3784,6 +3933,12 @@ function startOdysseusApp() {
       const hasText = messageInput && messageInput.value.trim().length > 0;
       const hasFiles = _hasAttachments();
 
+      if (sendBtn.dataset.mode === 'streaming') {
+        if (hasText) window.__odysseusQueueStreamingSubmit = Date.now();
+        handleSubmit(e);
+        return;
+      }
+
       // New chat mode — empty input, no attachments, no STT
       if (!hasText && !hasFiles && sendBtn.dataset.mode === 'newchat') {
         if (sessionModule) {
@@ -3823,9 +3978,10 @@ function startOdysseusApp() {
   // Enter to send (shift+enter for newline), or new chat when empty
   if (messageInput) {
     messageInput.addEventListener('keydown', (e) => {
-      const isMobile = window.innerWidth <= 768
+      if (e.defaultPrevented) return;
+      const isMobile = _isMobileChatInput();
 
-      if (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile) {
+      if (_shouldQueueFromMobileEnter(e, messageInput) || (e.key === 'Enter' && !e.shiftKey && !e.isComposing && !isMobile)) {
         e.preventDefault();
         // Flush the debounced icon update so dataset.mode reflects the current
         // text state. Without this, a fast type-and-Enter would still see the
@@ -3836,7 +3992,13 @@ function startOdysseusApp() {
           if (railNew) railNew.click();
           return;
         }
-        handleSubmit(e);
+        if (_isForegroundChatBusy() && messageInput.value && messageInput.value.trim()) {
+          if (chatModule && chatModule.queueStreamingComposerRequest && chatModule.queueStreamingComposerRequest()) {
+            return;
+          }
+          window.__odysseusQueueStreamingSubmit = Date.now();
+        }
+        _submitChatFormDirect(document.getElementById('chat-form'));
       }
     });
   }
@@ -3855,7 +4017,11 @@ function startOdysseusApp() {
     _syncModelPickerAutohide();
     messageInput.addEventListener('input', () => {
       _syncModelPickerAutohide();
-      _debouncedUpdateIcon();
+      if (sendBtn && sendBtn.dataset.mode === 'streaming') {
+        _updateSendBtnIcon();
+      } else {
+        _debouncedUpdateIcon();
+      }
     }, { passive: true });
   }
 
@@ -3910,14 +4076,13 @@ function startOdysseusApp() {
     _showDropHighlight();
   });
 
-  chatContainer.addEventListener('drop', (e) => {
+  chatContainer.addEventListener('drop', async (e) => {
     e.preventDefault();
     e.stopPropagation();
     _hideDropHighlight();
     const files = Array.from(e.dataTransfer.files);
     if (files.length === 0) return;
-    fileHandlerModule.addFiles(files);
+    await fileHandlerModule.addFiles(files);
-    fileHandlerModule.renderAttachStrip();
     uiModule.showToast(`Added ${files.length} file${files.length > 1 ? 's' : ''} to chat`);
   });
 
@@ -3934,12 +4099,13 @@ function startOdysseusApp() {
     attachStrip.style.borderRadius = '4px';
   });
   
-  attachStrip.addEventListener('drop', (e) => {
+  attachStrip.addEventListener('drop', async (e) => {
     e.preventDefault();
     attachStrip.style.backgroundColor = '';
     
     const files = Array.from(e.dataTransfer.files);
     if (files.length === 0) return;
+    await fileHandlerModule.addFiles(files);
 
     uiModule.showToast(`Added ${files.length} file${files.length > 1 ? 's' : ''} to chat`);
 
@@ -4007,14 +4173,13 @@ function startOdysseusApp() {
     if (_compareActive() && !e.relatedTarget) _hideCmpShield();
   }, true);
   window.addEventListener('dragend', _hideCmpShield, true);
-  window.addEventListener('drop', (e) => {
+  window.addEventListener('drop', async (e) => {
     if (!_isFileDrag(e) || !_compareActive()) return;
     e.preventDefault();
     _hideCmpShield();
     const files = Array.from(e.dataTransfer.files || []);
     if (!files.length) return;
-    fileHandlerModule.addFiles(files);
+    await fileHandlerModule.addFiles(files);
-    fileHandlerModule.renderAttachStrip();
     uiModule.showToast(`Added ${files.length} file${files.length > 1 ? 's' : ''} to attach`);
   }, true);
 
@@ -4049,24 +4214,40 @@ function startOdysseusApp() {
     console.error('Session module not loaded!');
   }
 
-  // Non-critical: load in parallel, resolve silently
+  const runNonCriticalStartup = (fn, delay = 4000) => {
-  modelsModule.refreshModels(false).then(() => {
+    let tries = 0;
-    try { sessionModule.updateModelPicker(); } catch (_) {}
+    const run = () => {
-    const modelsBox = document.getElementById('models');
+      const busy = !!window.__odysseusChatBusy
-    const hasModels = modelsBox && modelsBox.querySelector('.models-row');
+        || Date.now() < (window.__odysseusChatBusyUntil || 0)
-    if (!hasModels) {
+        || !!document.querySelector('.send-btn[data-mode="streaming"], .send-btn.send-pending');
-      const tip = document.getElementById('welcome-tip');
+      if (busy && tries < 12) {
-      if (tip) tip.textContent = 'Add an AI endpoint from Settings in the sidebar, or paste an endpoint/API key into the chat.';
+        tries += 1;
-    }
+        setTimeout(run, 2500);
-  }).catch(() => {});
+        return;
-  modelsModule.refreshProviders();
+      }
-  ragModule.loadPersonalDocs();
+      try { fn(); } catch (e) { console.warn('non-critical startup task failed:', e); }
-  memoryModule.loadMemories(); // Ensure memories are loaded on page load
+    };
-  
+    setTimeout(() => {
-  // Ensure the memory list is rendered after loading
+      if ('requestIdleCallback' in window) {
-  setTimeout(async () => {
+        window.requestIdleCallback(run, { timeout: 5000 });
-    await memoryModule.loadMemories();
+      } else {
-  }, 1000);
+        run();
+      }
+    }, delay);
+  };
+
+  // Non-critical startup work must not compete with first paint, chat send, or
+  // chat switching. Panels load their own data when opened; these are only warmups.
+  _syncWelcomeModelHint().catch(() => {});
+  runNonCriticalStartup(() => {
+    modelsModule.refreshModels(false).then(() => {
+      try { sessionModule.updateModelPicker(); } catch (_) {}
+      _syncWelcomeModelHint().catch(() => {});
+    }).catch(() => {});
+  }, 3500);
+  runNonCriticalStartup(() => modelsModule.refreshProviders(), 6500);
+  runNonCriticalStartup(() => ragModule.loadPersonalDocs(), 9000);
+  runNonCriticalStartup(() => memoryModule.loadMemories(), 12000);
   
   // Ensure proper initial state
   voiceRecorderModule.init();

static/index.html

@@ -1356,7 +1356,7 @@
         <div class="settings-sidebar">
           <!-- Section 1: AI plumbing (Add Models → AI Defaults → Search) -->
           <button class="settings-nav-item active" data-settings-tab="services">
-            <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="2" y="2" width="20" height="8" rx="2"/><rect x="2" y="14" width="20" height="8" rx="2"/><circle cx="6" cy="6" r="1"/><circle cx="6" cy="18" r="1"/></svg>
+            <svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.6" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14"/><path d="M5 12h14"/></svg>
             <span>Add Models</span>
           </button>
           <button class="settings-nav-item" data-settings-tab="added-models">
@@ -1979,34 +1979,6 @@
             </div>
           </div>
 
-          <div class="admin-card">
-            <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="m5 8 6 6"/><path d="m4 14 6-6 2-3"/><path d="M2 5h12"/><path d="M7 2h1"/><path d="m22 22-5-10-5 10"/><path d="M14 18h6"/></svg>Translation</h2>
-            <div class="settings-row" style="align-items:center;">
-              <div style="flex:1;min-width:0;">
-                <div class="settings-label" style="margin-bottom:3px;">Auto translate</div>
-                <div class="admin-toggle-sub" style="margin:0;">When an opened email appears to be in another language, prepare a translated view.</div>
-              </div>
-              <label class="admin-switch"><input type="checkbox" id="set-email-auto-translate"><span class="admin-slider"></span></label>
-            </div>
-            <div class="settings-row" style="align-items:center;margin-top:8px;">
-              <label class="settings-label" for="set-email-translate-language">Translate to</label>
-              <input id="set-email-translate-language" class="settings-select" list="set-email-translate-language-list" placeholder="English, Swedish, Japanese..." style="max-width:220px;">
-              <datalist id="set-email-translate-language-list">
-                <option value="English"></option>
-                <option value="Swedish"></option>
-                <option value="Norwegian"></option>
-                <option value="Danish"></option>
-                <option value="Japanese"></option>
-                <option value="Spanish"></option>
-                <option value="French"></option>
-                <option value="German"></option>
-                <option value="British English"></option>
-                <option value="Plain English"></option>
-              </datalist>
-              <span id="set-email-translate-msg" style="font-size:11px;margin-left:auto;"></span>
-            </div>
-          </div>
-
         </div>
 
         <!-- ═══ REMINDERS TAB ═══ -->
@@ -2122,7 +2094,7 @@
 
           <!-- ── Local card ─────────────────────────────────────────── -->
           <div class="admin-card">
-            <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><rect x="2" y="3" width="20" height="14" rx="2"/><path d="M8 21h8"/><path d="M12 17v4"/></svg>Add Local Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoint)</span>
+            <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.6" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M12 5v14"/><path d="M5 12h14"/></svg>Add Local Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoint)</span>
               <span style="flex:1"></span>
               <button class="admin-btn-sm" id="adm-epLocalTestBtn" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:4px;">
                 <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test
@@ -2154,7 +2126,7 @@
                     <input id="adm-epLocalUrl" type="text" placeholder="Paste endpoint URL, e.g. http://localhost:11434/v1" style="flex:1;min-width:0;border-top-left-radius:0;border-bottom-left-radius:0;">
                   </div>
                   <button class="admin-btn-add" id="adm-epLocalAddBtn" style="min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;flex-shrink:0;">
-                    <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>Add
+                    <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14"/><path d="M5 12h14"/></svg>Add
                   </button>
                 </div>
                 <div class="admin-model-form-row" id="adm-epLocalApiKey-row" style="display:none;">
@@ -2167,7 +2139,7 @@
 
           <!-- ── API card ───────────────────────────────────────────── -->
           <div class="admin-card">
-            <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><circle cx="12" cy="12" r="10"/><line x1="2" y1="12" x2="22" y2="12"/><path d="M12 2a15.3 15.3 0 0 1 4 10 15.3 15.3 0 0 1-4 10 15.3 15.3 0 0 1-4-10 15.3 15.3 0 0 1 4-10z"/></svg>Add API Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoint)</span>
+            <h2 style="display:flex;align-items:center;gap:8px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><circle cx="11" cy="12" r="8"/><path d="M19 5v6"/><path d="M16 8h6"/><path d="M7 12h8"/></svg>Add API Models <span style="opacity:0.45;font-weight:normal;font-size:0.82em">(Endpoint)</span>
               <span style="flex:1"></span>
               <button class="admin-btn-sm" id="adm-epApiTestBtn" style="font-size:11px;font-weight:normal;display:inline-flex;align-items:center;gap:4px;">
                 <svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3"/></svg>Test
@@ -2236,7 +2208,7 @@
                     <input id="adm-epApiKey" type="password" placeholder="API key, e.g. sk-proj-AbCdEf…" autocomplete="off" style="flex:1;padding-left:28px;height:32px;box-sizing:border-box;">
                   </div>
                   <button class="admin-btn-add" id="adm-epAddBtn" style="height:32px;min-width:55px;text-align:center;display:inline-flex;align-items:center;justify-content:center;gap:4px;flex-shrink:0;box-sizing:border-box;">
-                    <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>Add
+                    <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><path d="M12 5v14"/><path d="M5 12h14"/></svg>Add
                   </button>
                 </div>
                 <div id="adm-epApiMsg" class="adm-ep-inline-msg"></div>

static/js/MODULE_SUMMARY.md

@@ -1,124 +1,228 @@
-# Module Organization Summary
+# Frontend Module Organization Summary
 
-## Purpose
+> **Scope:** This document describes the architecture of the Odysseus no-build
-This document describes what each JavaScript module is responsible for.
+> frontend. The app is a collection of native ES6 modules loaded from
-
+> `static/`. The authoritative source is the current `static/js/` tree and the
-> **Note:** This file is a partial, historical overview — not a complete authoritative
+> top-level orchestrator `static/app.js`.
-> inventory. The authoritative module set is the current `static/js/` tree plus the
-> scripts loaded by `static/index.html`. As of this writing that tree holds **65 `.js`
-> files** across **8 subdirectories** (`calendar/`, `color/`, `compare/`, `editor/`,
-> `emailLibrary/`, `markdown/`, `research/`, `util/`), and `static/index.html` loads
-> **35** `/static…` script tags. The catalog below covers only the original core
-> modules and is not kept in sync with every module.
 
 ---
 
-## Core Modules (in static/js/)
+## 1. Top-level Application Orchestrator
 
-### 1. **ui.js**
+### `static/app.js`
-- UI helper functions and utilities
+*Main application entry point.*
-- Toast notifications (`showToast`, `showError`)
-- Element getter (`el()`)
-- Clipboard operations (`copyToClipboard`)
-- Scroll management (`scrollHistory`, `setAutoScroll`)
-- Auto-resize textarea
-- Debounce utility
 
-### 2. **markdown.js**
+- Imports all feature modules.
-- Markdown processing and rendering
+- Exposes a few modules on `window` for legacy inter-module reachability
-- Convert markdown to HTML (`mdToHtml`)
+  (`themeModule`, `sessionModule`, `uiModule`, `adminModule`, `cookbookModule`).
-- Code block handling with syntax highlighting
+- Patches `fetch` so any `401` redirects the user to `/login`.
-- Content rendering for message arrays
+- Fetches the default chat configuration and handles deep-link route openers
-- Text cleanup (`squashOutsideCode`)
+  (`/notes`, `/calendar`, `/email`, `/memory`, `/gallery`, `/cookbook`, `/library`, `/tasks`).
+- Wires global event listeners: chat-history scrolling, popups, Escape handling,
+  drag-and-drop/paste attachment handling, transcription export, sidebar toggles,
+  rail/tool buttons, and session sorting.
+- Loads auth status and applies per-user privilege restrictions.
 
-### 3. **sessions.js**
+### `static/index.html`
-- Session/chat management
+*SPA shell.* Loads `app.js` as a module, includes the theme-aware inline script,
-- Create, load, delete, switch sessions
+and defines the DOM skeleton that the modules populate (chat history, composer,
-- Session history loading
+sidebar, icon rail, modals).
-- Direct chat creation with models
-- Session renaming
-
-### 4. **memory.js**
-- AI memory management
-- Load, add, edit, delete memories
-- Memory search/filtering
-- Memory UI rendering
-- Memory count updates
-
-### 5. **fileHandler.js**
-- File attachment handling
-- File picker dialog
-- File upload to server
-- Attachment strip rendering
-- Pending files management
-- File preview/removal
-
-### 6. **voiceRecorder.js**
-- Voice recording functionality
-- Start/stop recording
-- Audio file creation
-- Microphone permission handling
-- Recording UI updates
-
-### 7. **models.js**
-- Model scanning and display
-- Local model discovery (ports 8000-8020)
-- Provider management (OpenAI)
-- Model selection UI
-
-### 8. **rag.js**
-- RAG (Retrieval Augmented Generation) management
-- Load personal documents
-- Add directories to RAG
-- Display included files/directories
-
-### 9. **presets.js**
-- Conversation preset management
-- Load, save, activate presets
-- Custom preset configuration
-- Temperature, tokens, system prompt settings
-
-### 10. **search.js**
-- Web search settings
-- Provider selection (DuckDuckGo, Brave, SearXNG)
-- API key management
-- Save/load search configuration
-
-### 11. **chat.js** ⭐ (The Big One)
-- Main chat functionality
-- Message handling (`addMessage`)
-- Chat submission (`handleChatSubmit`)
-- Streaming response handling
-- Performance metrics display
-- Abort request management
-- Loading states and error handling
 
 ---
 
-## Main Application File
+## 2. Core Foundation Modules
 
-### **app.js**
+These are imported first and used across most features.
-- Application initialization
+
-- Event listener setup
+| Module | Primary Exports | Responsibility |
-- Drag & drop handlers
+|---|---|---|
-- Keyboard shortcuts
+| **`ui.js`** | `showToast`, `showError`, `el`, `copyToClipboard`, `scrollHistory`, `setAutoScroll`, `autoResize`, `debounce`, `esc` | Shared UI helpers, toast notifications, scroll behavior, element accessor, text escaping. |
-- Module initialization
+| **`storage.js`** | `default` storage wrapper | LocalStorage helpers and toggle state persistence. |
-- Global configuration (API_BASE)
+| **`markdown.js`** | `mdToHtml`, `processWithThinking`, `squashOutsideCode`, `normalizeThinkingMarkup`, `extractThinkingBlocks`, `hasUnclosedThinkTag`, `startsWithReasoningPrefix` | Markdown→HTML, thinking/reasoning block parsing, code-block normalization. |
-- Coordinates all modules together
+| **`spinner.js`** | `create`, `createWhirlpool` | Loading/spinner factories for streaming and tool cards. |
+| **`keyboard-shortcuts.js`** | `initKeyboardShortcuts` | Global keyboard shortcut wiring. |
+| **`sidebar-layout.js`** | `initSidebarLayout`, `syncRailSide` | Wide sidebar ↔ icon-rail layout behavior. |
+| **`section-management.js`** | `initSectionCollapse`, `initSectionDrag` | Collapsible/draggable sidebar sections. |
+| **`modalManager.js`** | side-effect import | Unified minimize/restore behavior for floating tool modals. |
+| **`tileManager.js`** | side-effect import | Desktop window tiling and snap-to-edge behavior. |
+| **`windowDrag.js`** | `makeWindowDraggable` | Drag support for floating panels. |
+| **`modalSnap.js`**, **`toolWindowZOrder.js`**, **`windowResize.js`** | — | Modal snapping, z-index management, resize handles. |
 
 ---
 
-## Dependency Order (Load Order in HTML)
+## 3. Chat Pipeline
-```html
+
-<script src="/static/js/sessions.js"></script>      <!-- 1. Sessions first -->
+The largest and most central subsystem. Chat submission → backend SSE → progressive rendering of text, tools, research, documents, and UI events.
-<script src="/static/js/memory.js"></script>        <!-- 2. Memory -->
+
-<script src="/static/js/markdown.js"></script>      <!-- 3. Markdown -->
+| Module | Responsibility |
-<script src="/static/js/ui.js"></script>            <!-- 4. UI utilities -->
+|---|---|
-<script src="/static/js/fileHandler.js"></script>   <!-- 5. File handling -->
+| **`chat.js`** | Main chat controller. Handles `handleChatSubmit`, stops/continues, builds `FormData`, posts to `/api/chat_stream`, reads the SSE stream, and dispatches each JSON event to the appropriate renderer. Tracks background streams, stalls, auto-recovery, and multi-round agent state. |
-<script src="/static/js/voiceRecorder.js"></script> <!-- 6. Voice -->
+| **`chatStream.js`** | Helpers shared between streaming consumers: browser notifications, background-stream completion toasts, and `ui_control` event handling. |
-<script src="/static/js/models.js"></script>        <!-- 7. Models -->
+| **`chatRenderer.js`** | Message DOM construction: `addMessage`, role labels, model route labels, color coding, footers, metrics, code blocks, sources boxes (`web`/`research`/`RAG`), findings box, images, report links, ask-user cards, welcome screen, and transcript utilities. |
-<script src="/static/js/rag.js"></script>           <!-- 8. RAG -->
+| **`streamingRenderer.js`** | Incremental streaming renderer used by `chat.js`. Freezes finalized DOM blocks and only re-renders the growing tail to avoid flicker and O(N²) re-parsing. |
-<script src="/static/js/presets.js"></script>       <!-- 9. Presets -->
+| **`streamingSegmenter.js`** | Splits a token stream into display units (text vs code fences) for `streamingRenderer.js`. |
-<script src="/static/js/search.js"></script>        <!-- 10. Search -->
+| **`slashCommands.js`** | Slash-command registry (`/help`, `/setup`, etc.), parsing, and dispatch handlers. Exported functions are consumed by `chat.js` and `slashAutocomplete.js`. |
-<script src="/static/js/chat.js"></script>          <!-- 11. Chat -->
+| **`slashAutocomplete.js`** | Composer autocomplete popup for `/` commands. |
-<script src="/static/app.js"></script>              <!-- 12. Main app LAST -->
+| **`composerArrowUpRecall.js`** | Recall last user message with `↑` on an empty composer. |
+| **`assistant.js`** | Assistant/persona behaviors and message styling helpers. |
+| **`tts-ai.js`** | AI text-to-speech manager, enqueueing, streaming TTS, and playback button injection. |
+| **`voiceRecorder.js`** | Voice recording from the composer microphone. |
+| **`fileHandler.js`** | Attachment picker, paste/drop handling, upload, attachment strip rendering, pending-file management. |
+| **`codeRunner.js`** | Client-side execution affordances for code blocks returned by the model. |
+
+---
+
+## 4. Model, Endpoint, and Configuration Modules
+
+| Module | Responsibility |
+|---|---|
+| **`models.js`** | Model discovery / scanning, local model port probing, provider management, model selection UI state. |
+| **`modelPicker.js`** | Composer model-picker dropdown and endpoint selection. |
+| **`modelSort.js`** | Sorting helpers for model lists. |
+| **`model/matchKey.js`** | Model-to-key matching helper. |
+| **`providers.js`** | Provider metadata and account-management helpers. |
+| **`providerDeviceFlow.js`** | OAuth device-flow support for providers. |
+| **`presets.js`** | Character/preset selection, custom preset saving, inject prefix/suffix handling. |
+| **`search.js`** | Web-search settings, provider selection, API key management. |
+| **`settings.js`** | Settings panel (models, search, appearance, users, MCP, RAG, embedding, tokens). |
+| **`admin.js`** | Admin panel and privileged user/endpoint configuration. |
+| **`theme.js`** | Theme presets, custom colors, fonts, backgrounds, live theme switching. |
+
+---
+
+## 5. Session, Sidebar, and Workspace
+
+| Module | Responsibility |
+|---|---|
+| **`sessions.js`** | Chat session list loading, creation, switching, renaming, archiving, library modal, and direct-chat creation. Tracks current session, streaming/research indicators in the sidebar. |
+| **`workspace.js`** | Workspace folder path management for shell/file tool confinement. |
+| **`search-chat.js`** | In-chat history search. |
+| **`skills.js`** | Client-side skill library UI (load, edit, delete, test, and audit status display). |
+
+---
+
+## 6. Knowledge, Memory, and RAG
+
+| Module | Responsibility |
+|---|---|
+| **`memory.js`** | AI memory CRUD, search/filter UI, memory extraction, count badge. |
+| **`rag.js`** | Personal document RAG: load documents, add directories/files, show included paths. |
+| **`group.js`** | Group-chat UI and model orchestration. |
+
+---
+
+## 7. Document and Editor Subsystems
+
+| Module | Responsibility |
+|---|---|
+| **`document.js`** | Tabbed document editor, AI edit suggestions, Markdown/HTML/CSV editing, document streaming (`streamDocOpen`/`streamDocDelta`), and panel state. |
+| **`documentLibrary.js`** | Document library modal. |
+| **`editor/`** | Gallery image editor canvas modules: layers, brush, inpaint, crop, filters, state, history panel, top-bar wiring, canvas coordinate helpers, and AI model runners for inpainting/background-removal. |
+
+---
+
+## 8. Research UI
+
+| Module | Responsibility |
+|---|---|
+| **`research/panel.js`** | Research panel UI, job list, and controls. |
+| **`research/jobs.js`** | Research job polling and status rendering. |
+| **`researchSynapse.js`** | Animated research-progress visualization shown inside the chat bubble during a research run. |
+
+---
+
+## 9. Gallery, Email, Calendar, Tasks, and Notes
+
+| Module | Responsibility |
+|---|---|
+| **`gallery.js`** / **`galleryEditor.js`** | Gallery/image library and canvas editor entry points. |
+| **`emailInbox.js`** / **`emailLibrary.js`** | Email inbox reader and library modal. Sub-modules handle signatures, reply recipients, state, and signature folding. |
+| **`calendar.js`** / **`calendar/utils.js`** / **`calendar/reminders.js`** | Calendar views, event forms, reminders. |
+| **`tasks.js`** | Scheduled task/recurring LLM job UI. |
+| **`notes.js`** | Notes and todo panel, reminders, pinboard. |
+
+---
+
+## 10. Cookbook (Model Serving)
+
+| Module | Responsibility |
+|---|---|
+| **`cookbook.js`** | Cookbook main UI: hardware fitting, presets, action panels. |
+| **`cookbook-hwfit.js`** / **`cookbook-diagnosis.js`** / **`cookbook-deps-recipes.js`** | Hardware-fit scoring, dependency diagnosis, recipe handling. |
+| **`cookbookDownload.js`** / **`cookbookServe.js`** / **`cookbookRunning.js`** / **`cookbookSchedule.js`** / **`cookbookPorts.js`** / **`cookbookProgressSignal.js`** | Model download/serve flow, running job cards, scheduling, port detection, and progress computation. |
+
+---
+
+## 11. Compare and Utility Modules
+
+| Module | Responsibility |
+|---|---|
+| **`compare/index.js`** (with `compare/state.js`, `compare/stream.js`, `compare/panes.js`, `compare/selector.js`, `compare/scoreboard.js`, `compare/probe.js`, `compare/vote.js`, `compare/icons.js`) | Model compare mode: parallel streams, panes, scoring, vote UI. |
+| **`censor.js`** | Text/image censor overlay toggles. |
+| **`a11y.js`** | Accessibility helpers. |
+| **`platform.js`** | Platform detection (macOS/Windows/Linux) and keyboard-modifier helpers. |
+| **`escMenuStack.js`** | Stack manager for dismissible popups. |
+| **`dragSort.js`** | Drag-to-sort shared behavior. |
+| **`tourHints.js`** / **`tourAutoplay.js`** | Onboarding tour helpers. |
+| **`color/hex.js`**, **`colorPicker.js`**, **`langIcons.js`**, **`util/ordinal.js`** | Small utility modules for color, language icons, and formatting. |
+
+---
+
+## 12. Frontend Event Streaming Flow
+
+```
+User submits composer
+  └── chat.js::handleChatSubmit() builds FormData
+        ├── fileHandler.uploadPending() for attachments
+        ├── document.js saved (if a document panel is open)
+        └── POST /api/chat_stream
+
+Server responds with SSE stream
+  └── chat.js reads chunks via res.body.getReader() + TextDecoder
+        ├── Lines starting with "event:" set next-error state
+        └── Lines starting with "data:" carry JSON payloads
+
+JSON events are dispatched by "type":
+  delta              → streamingRenderer → markdown → live reply text
+  agent_prep         → update spinner label
+  tool_start         → finalize text bubble; create agent-thread node with wave animation
+  tool_progress      → append/update live stdout/stderr tail
+  tool_output        → mark node done/failed, render output, diffs, screenshots
+  agent_step         → finalize tool thread; create new msg-continuation bubble
+  doc_stream_open    → document.js opens a live document
+  doc_stream_delta   → document.js appends content to that document
+  research_progress  → researchSynapse visualization + spinner timer
+  research_sources   → build sources box for research
+  research_done      → reload session history to show the report
+  web_sources        → build web-search sources box
+  model_info         → update role header with requested/actual model
+  fallback           → show fallback model toast + update role label
+  metrics            → collect/display token/cost metrics
+  message_saved      → store database id on the message element
+  budget_exceeded    → show budget banner
+  rounds_exhausted   → show Continue button for step-limit hits
+  teacher_takeover   → insert escalation banner, reset round state
+  skill_saved        → show skill-learned banner
+```
+
+Foreground vs background streams:
+- If the user switches sessions while a stream is running, `chat.js` pauses DOM
+  updates and stores the state in `_backgroundStreams`. Completion is signaled
+  with a sidebar dot/notifications, and the history is reloaded when the user
+  returns.
+
+---
+
+## 13. What Changed from the Previous Summary
+
+- The frontend is now exclusively ES6-module based; the old `<script>` tag load
+  order is no longer authoritative.
+- `chat.js` is the streaming controller, but message rendering has been split
+  into `chatRenderer.js`, `streamingRenderer.js`, `chatStream.js`, and `researchSynapse.js`.
+- New major subsystems added since the original summary: compare mode
+  (`compare/`), document editor streaming (`document.js`), research UI
+  (`research/`), model cookbook (`cookbook*.js`), group chat (`group.js`),
+  voice/TTS (`voiceRecorder.js`, `tts-ai.js`), skill UI (`skills.js`), and
+  slash autocomplete (`slashAutocomplete.js`).
+- `sessions.js` now owns sidebar session state, streaming/research indicators,
+  and the library/archive modals.

static/js/admin.js

@@ -471,7 +471,7 @@ async function loadEndpoints() {
   const listLegacy = el('adm-epList');
   // Refresh model picker so new endpoints show up in chat
   if (window.modelsModule && window.modelsModule.refreshModels) {
-    window.modelsModule.refreshModels();
+    window.modelsModule.refreshModels(true);
     setTimeout(() => {
       if (window.sessionModule && window.sessionModule.updateModelPicker) {
         window.sessionModule.updateModelPicker();

static/js/chat.js

@@ -48,9 +48,53 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     try {
       window.__odysseusChatBusy = !!active;
       window.__odysseusChatBusyUntil = active ? Date.now() + 120000 : Date.now() + 1200;
+      window.dispatchEvent(new CustomEvent('odysseus:chat-busy-change', { detail: { active: !!active } }));
     } catch (_) {}
   }
   let _pendingContinue = null; // Stores the stopped AI element to merge with new response
+  function _createChatSendPerf() {
+    const started = (performance && performance.now) ? performance.now() : Date.now();
+    let last = started;
+    let reported = false;
+    const stages = [];
+    const now = () => (performance && performance.now) ? performance.now() : Date.now();
+    return {
+      mark(name) {
+        const t = now();
+        stages.push({ name, delta_ms: Math.round(t - last), at_ms: Math.round(t - started) });
+        last = t;
+      },
+      report(extra) {
+        if (reported) return;
+        const total = Math.round(now() - started);
+        const slowStage = stages.some(s => (s.delta_ms || 0) >= 1500);
+        if (total < 1500 && !slowStage) return;
+        reported = true;
+        const payload = JSON.stringify({
+          type: 'chat_send',
+          total_ms: total,
+          stages,
+          extra: extra || '',
+          session: sessionModule && sessionModule.getCurrentSessionId ? sessionModule.getCurrentSessionId() : '',
+        });
+        try {
+          if (navigator.sendBeacon) {
+            navigator.sendBeacon(`${API_BASE}/api/client-perf`, new Blob([payload], { type: 'application/json' }));
+            return;
+          }
+        } catch (_) {}
+        try {
+          fetch(`${API_BASE}/api/client-perf`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: payload,
+            keepalive: true,
+            credentials: 'same-origin',
+          }).catch(() => {});
+        } catch (_) {}
+      }
+    };
+  }
   // ── Auto-recovery: when a turn's stream silently dies (connection drop) or
   // goes quiet while the connection is alive, re-engage the model with a
   // completion handshake instead of leaving it hung. Capped so it can't loop.
@@ -120,6 +164,26 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     return final && !visible ? 'Done.' : visible;
   }
 
+  function _stripIncompleteRawToolJsonForChat(text) {
+    const s = String(text || '');
+    const starts = ['[{"function"', '[\n{"function"', '{"function"'];
+    let idx = -1;
+    for (const marker of starts) idx = Math.max(idx, s.lastIndexOf(marker));
+    if (idx < 0) return s;
+    const tail = s.slice(idx);
+    // Complete raw OpenAI-style function blobs are removed by stripToolBlocks.
+    // While the stream is still mid-JSON, hide the tail so it never flashes in
+    // the chat bubble as prose.
+    if (!/"type"\s*:\s*"function"/.test(tail) || !/\}\s*\]?\s*(?:<\/?\|(?:assistant|assistan|user|system|tool)\|>?)?\s*$/i.test(tail)) {
+      return s.slice(0, idx);
+    }
+    return s;
+  }
+
+  function _streamDisplayText(text, opts = {}) {
+    return stripToolBlocks(_stripIncompleteRawToolJsonForChat(_stripDocumentFenceForChat(text, opts)));
+  }
+
   function _showDocumentWritingStatus(contentEl) {
     const msg = contentEl && contentEl.closest ? contentEl.closest('.msg') : null;
     const chatBox = document.getElementById('chat-history');
@@ -204,6 +268,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
   let _streamSessionId = null; // Session ID for the currently active reader loop
   let _lastReaderActivity = 0; // Timestamp of last reader.read() success — used to detect frozen streams
   let _webLockRelease = null;  // Function to release the Web Lock held during streaming
+  let _staleStreamProbeInFlight = false;
+  const STALE_LOCAL_STREAM_MS = 15000;
 
   /** Check if an SSE reader is still actively connected for a session. */
   function hasActiveStream(sessionId) {
@@ -326,7 +392,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       // arrow out for the stop icon — otherwise the swap happens mid-flight
       // and the user sees nothing fly out.
       setTimeout(() => {
-        submitBtn.innerHTML = _stopSvg;
+        if (submitBtn.dataset.mode !== 'streaming') return;
+        const msgInput = uiModule.el('message');
+        const hasQueuedText = !!(msgInput && msgInput.value && msgInput.value.trim());
+        submitBtn.innerHTML = hasQueuedText && icons ? icons.send : _stopSvg;
+        submitBtn.dataset.phase = hasQueuedText ? 'queue' : 'processing';
+        submitBtn.title = hasQueuedText ? 'Queue message' : 'Stop generation';
         submitBtn.classList.remove('anim-launch');
         void submitBtn.offsetWidth;
         submitBtn.classList.add('anim-land');
@@ -471,6 +542,24 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     return true;
   }
 
+  export function queueStreamingComposerRequest() {
+    if (!isStreaming) return false;
+    const queuedInput = uiModule.el('message');
+    const queuedText = (queuedInput && queuedInput.value || '').trim();
+    if (!queuedText) return false;
+    if (fileHandlerModule.getPendingCount && fileHandlerModule.getPendingCount()) {
+      try { uiModule.showError && uiModule.showError('Finish the current response before queueing messages with attachments.'); } catch (_) {}
+      return true;
+    }
+    if (_queueAgentRequest(queuedText)) {
+      queuedInput.value = '';
+      queuedInput.dispatchEvent(new Event('input', { bubbles: true }));
+      if (uiModule.autoResize) uiModule.autoResize(queuedInput);
+      try { window._updateSendBtnIcon && window._updateSendBtnIcon(); } catch (_) {}
+    }
+    return true;
+  }
+
   function _drainQueuedAgentRequests() {
     if (isStreaming || _sendInFlight || !_queuedAgentRequests.length) return;
     if (_queuedDrainTimer) return;
@@ -507,21 +596,13 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       return;
     }
 
-    // If currently streaming, a non-empty composer means "queue this next".
+    // If currently streaming, keyboard Enter can queue a non-empty composer.
-    // Empty composer keeps the existing Stop behavior.
+    // Clicking the stop icon should still stop normally, even if text exists.
     if (isStreaming) {
-      const queuedInput = uiModule.el('message');
+      const queueRequestedAt = Number(window.__odysseusQueueStreamingSubmit || 0);
-      const queuedText = (queuedInput && queuedInput.value || '').trim();
+      const shouldQueueStreamingSubmit = queueRequestedAt && Date.now() - queueRequestedAt < 1200;
-      if (queuedText) {
+      window.__odysseusQueueStreamingSubmit = 0;
-        if (fileHandlerModule.getPendingCount && fileHandlerModule.getPendingCount()) {
+      if (shouldQueueStreamingSubmit && queueStreamingComposerRequest()) {
-          try { uiModule.showError && uiModule.showError('Finish the current response before queueing messages with attachments.'); } catch (_) {}
-          return;
-        }
-        if (_queueAgentRequest(queuedText)) {
-          queuedInput.value = '';
-          queuedInput.dispatchEvent(new Event('input', { bubbles: true }));
-          if (uiModule.autoResize) uiModule.autoResize(queuedInput);
-        }
         return;
       }
       if (fileHandlerModule.isUploading && fileHandlerModule.isUploading()) {
@@ -652,6 +733,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
     // --- Send-path entry: block re-clicks between submit and stream start ---
     if (_sendInFlight) return;
+    const _sendPerf = _createChatSendPerf();
     _sendInFlight = true;
     _setForegroundChatBusy(true);
     // Instant visual feedback so the user sees their click was accepted
@@ -710,18 +792,34 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
     // Materialize pending session (deferred from model click) on first message
     if (sessionModule.hasPendingChat && sessionModule.hasPendingChat()) {
+      _sendPerf.mark('pending_session_begin');
       const ok = await sessionModule.materializePendingSession();
+      _sendPerf.mark('pending_session_done');
       if (!ok || !sessionModule.getCurrentSessionId()) { _releaseSendFlag(); return; }
     }
 
+    if (!sessionModule.getCurrentSessionId()) {
+      // Auto-create a session using default chat config. Always fetch fresh
+      // so that a recent Settings change takes effect without a page reload.
+      try {
+        const pending = sessionModule.getPendingChat && sessionModule.getPendingChat();
+        if (pending && pending.url && pending.modelId) {
+          const ok = await sessionModule.materializePendingSession();
+          if (!ok || !sessionModule.getCurrentSessionId()) { _releaseSendFlag(); return; }
+        }
+      } catch (_) {}
+    }
+
     if (!sessionModule.getCurrentSessionId()) {
       // Auto-create a session using default chat config. Always fetch fresh
       // so that a recent Settings change takes effect without a page reload.
       try {
         let dc = null;
         try {
+          _sendPerf.mark('default_chat_fetch_begin');
           const dcRes = await fetch('/api/default-chat');
           dc = await dcRes.json();
+          _sendPerf.mark('default_chat_fetch_done');
           if (dc && dc.endpoint_url && dc.model) {
             try { window.__odysseusDefaultChat = dc; } catch (_) {}
           }
@@ -729,8 +827,11 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
           dc = (typeof window !== 'undefined' && window.__odysseusDefaultChat) || null;
         }
         if (dc.endpoint_url && dc.model) {
+          _sendPerf.mark('direct_chat_create_begin');
           await sessionModule.createDirectChat(dc.endpoint_url, dc.model, dc.endpoint_id);
+          _sendPerf.mark('direct_chat_create_done');
           const ok = await sessionModule.materializePendingSession();
+          _sendPerf.mark('direct_chat_materialize_done');
           if (!ok || !sessionModule.getCurrentSessionId()) { _releaseSendFlag(); return; }
         } else {
           el('message').value = '';
@@ -886,6 +987,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       if (!skipBubble) {
         _userMsgEl = addMessage('user', userDisplay, null, _pendingAttachInfo ? { attachments: _pendingAttachInfo } : null);
       }
+      _sendPerf.mark('user_bubble_visible');
       messageInput.value = '';
       messageInput.style.height = '';
       messageInput.dispatchEvent(new Event('input'));
@@ -921,9 +1023,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
       let ids = [];
       try {
+        _sendPerf.mark('upload_begin');
         ids = await fileHandlerModule.uploadPending({ sessionId: sessionModule.getCurrentSessionId() });
+        _sendPerf.mark('upload_done');
       } catch(e) {
         console.error('upload failed', e);
+        _sendPerf.mark('upload_failed');
       }
       if (_pendingAttachInfo && !ids.length && !(_pendingRegenAttachments && _pendingRegenAttachments.length)) {
         if (_userMsgEl && _userMsgEl.parentNode) _userMsgEl.remove();
@@ -1021,11 +1126,18 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       let activeDocIdForSend = documentModule && typeof documentModule.getCurrentDocId === 'function'
         ? documentModule.getCurrentDocId()
         : null;
-      if (!activeDocIdForSend && activeEmailComposerCtx?.docId) {
+      if (activeEmailComposerCtx?.docId) {
         activeDocIdForSend = activeEmailComposerCtx.docId;
       }
       if (documentModule && activeDocIdForSend) {
-        try { await documentModule.saveDocument(); } catch(e) { console.warn('doc auto-save failed', e); }
+        try {
+          _sendPerf.mark('doc_save_begin');
+          await documentModule.saveDocument();
+          _sendPerf.mark('doc_save_done');
+        } catch(e) {
+          console.warn('doc auto-save failed', e);
+          _sendPerf.mark('doc_save_failed');
+        }
       }
 
       // Inject document selection context if present
@@ -1057,7 +1169,13 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       if (ids.length) fd.append('attachments', JSON.stringify(ids));
       // Auto-save & send active doc ID so the backend sees latest content
       if (documentModule && activeDocIdForSend) {
-        try { await documentModule.saveDocument({ silent: true }); } catch (_e) { /* best-effort */ }
+        try {
+          _sendPerf.mark('doc_silent_save_begin');
+          await documentModule.saveDocument({ silent: true });
+          _sendPerf.mark('doc_silent_save_done');
+        } catch (_e) {
+          _sendPerf.mark('doc_silent_save_failed');
+        }
         fd.append('active_doc_id', activeDocIdForSend);
       }
       // Active email context — when an email reader is open, pass its
@@ -1076,7 +1194,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
           if (emCtx.account) fd.append('active_email_account', String(emCtx.account));
         }
       } catch (_e) { /* best-effort */ }
-      // Web toggle: pre-search in Chat mode, tool permission in Agent mode
+      // Web toggle: pre-search in Chat mode only. Agent mode should not
+      // opportunistically hit SearXNG just because the chat search toggle is
+      // on; explicit web/current-info requests are handled by the backend
+      // intent gate.
       const toggleState = Storage.loadToggleState();
       let isAgentMode = (toggleState.mode || 'chat') === 'agent';
       const incognitoChk = el('incognito-toggle');
@@ -1088,13 +1209,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       }
       fd.append('mode', isAgentMode ? 'agent' : 'chat');
       if (el('web-toggle').checked) {
-        if (isAgentMode) {
+        if (!isAgentMode) {
-          fd.append('allow_web_search', 'true');
-        } else {
           fd.append('use_web', 'true');
         }
-      } else if (isAgentMode) {
+      }
-        fd.append('allow_web_search', 'false');
+      if (isAgentMode) {
+        fd.append('allow_web_search', el('web-toggle').checked ? 'true' : 'false');
       }
       if (el('research-toggle').checked) {
         fd.append('use_research', 'true');
@@ -1229,12 +1349,15 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         try { return Intl.DateTimeFormat().resolvedOptions().timeZone || ''; }
         catch { return ''; }
       })();
+      _sendPerf.mark('chat_stream_post_begin');
       const res = await fetch(`${API_BASE}/api/chat_stream`, {
         method: 'POST',
         body: fd,
         headers: { 'X-Tz-Offset': String(_tzOffsetMin), 'X-Tz-Name': _tzName },
         signal: abortCtrl.signal
       });
+      _sendPerf.mark('chat_stream_headers');
+      _sendPerf.report('headers_received');
       
       if (!res.ok) {
         clearResponseTimeout();
@@ -1280,6 +1403,8 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       if (_chatLog) _chatLog.setAttribute('aria-busy', 'true');
 
       const reader = res.body.getReader();
+      _sendPerf.mark('reader_ready');
+      _sendPerf.report('reader_ready');
       const decoder = new TextDecoder();
       let buffer = '';
       let metrics = null;
@@ -1322,6 +1447,32 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         }
         return contentDiv;
       }
+      function _ensureVisibleRoundForDelta() {
+        if (!roundHolder || roundHolder.style.display !== 'none') return;
+        const box = document.getElementById('chat-history');
+        if (!box) {
+          roundHolder.style.display = '';
+          return;
+        }
+        const newWrap = document.createElement('div');
+        newWrap.className = 'msg msg-ai msg-continuation streaming';
+        const newRole = document.createElement('div');
+        newRole.className = 'role';
+        const metaS = sessionModule.getSessions().find(s => s.id === streamSessionId);
+        const requested = holder?._requestedModel || metaS?.model || modelName;
+        const actual = holder?._actualModel || requested;
+        newRole.textContent = _modelRouteLabel(requested, actual) || '';
+        _applyModelColor(newRole, actual);
+        newWrap.appendChild(newRole);
+        const newBody = document.createElement('div');
+        newBody.className = 'body';
+        newWrap.appendChild(newBody);
+        box.appendChild(newWrap);
+        if (lastToolThread && lastToolThread.isConnected) lastToolThread.classList.add('has-bottom');
+        roundHolder = newWrap;
+        roundText = '';
+        roundFinalized = false;
+      }
       const esc = uiModule.esc;
       // Remove thinking spinner helper
       _removeThinkingSpinner = () => {
@@ -1445,7 +1596,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
       // Direct render helper for streaming text
       _renderStream = () => {
-        let dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(_stripDocumentFenceForChat(roundText)));
+        let dt = markdownModule.normalizeThinkingMarkup(_streamDisplayText(roundText));
         const bodyEl = roundHolder.querySelector('.body');
         const contentEl = _ensureStreamLayout(bodyEl);
 
@@ -1671,7 +1822,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 typewriterInto(roundHolder.querySelector('.body'), errMsg);
                 break;
               }
-              if (json.delta || json.type === 'agent_prep' || json.type === 'tool_start' || json.type === 'tool_output' || json.type === 'tool_progress' || json.type === 'agent_step' || json.type === 'doc_stream_open' || json.type === 'doc_stream_delta' || json.type === 'research_progress') {
+              if (json.delta || json.type === 'agent_prep' || json.type === 'tool_start' || json.type === 'tool_output' || json.type === 'tool_progress' || json.type === 'agent_step' || json.type === 'loop_breaker_triggered' || json.type === 'intent_nudge_exhausted' || json.type === 'doc_stream_open' || json.type === 'doc_stream_delta' || json.type === 'research_progress') {
                 clearResponseTimeout();
                 clearProcessingProbe();
                 clearFirstTokenWaitTimers();
@@ -1700,24 +1851,25 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                   if (!_thinkOpen) { _delta = '<think>' + _delta; _thinkOpen = true; }
                 } else if (_thinkOpen) {
                   _delta = '</think>' + _delta; _thinkOpen = false;
-                }
+	                }
-                const wasEmpty = !accumulated;
+	                const wasEmpty = !accumulated;
-                accumulated += _delta;
+	                accumulated += _delta;
-                roundText += _delta;
+	                currentAccumulated = accumulated; // Update global tracker
-                currentAccumulated = accumulated; // Update global tracker
+	                // First token arrived — switch stop button from processing to streaming
-                // First token arrived — switch stop button from processing to streaming
+	                if (wasEmpty && submitBtn && !_isBg) {
-                if (wasEmpty && submitBtn && !_isBg) {
+	                  submitBtn.dataset.phase = 'receiving';
-                  submitBtn.dataset.phase = 'receiving';
                 }
 
                 // Update background map if running in background
                 if (_isBg) {
                   var bgEntry = _backgroundStreams.get(streamSessionId);
-                  if (bgEntry) bgEntry.accumulated = accumulated;
+	                  if (bgEntry) bgEntry.accumulated = accumulated;
-                  continue; // Skip all DOM writes
+	                  continue; // Skip all DOM writes
-                }
+	                }
+	                _ensureVisibleRoundForDelta();
+	                roundText += _delta;
 
-                // --- Text-fence doc streaming (for models that don't use native tool calls) ---
+	                // --- Text-fence doc streaming (for models that don't use native tool calls) ---
                 if (!_docFenceOpened && documentModule && (roundText.includes('```create_document\n') || roundText.includes('```document\n') || roundText.includes('```documen\n'))) {
                   const fenceMarker = roundText.includes('```document\n') ? '```document\n' : (roundText.includes('```documen\n') ? '```documen\n' : '```create_document\n');
                   const fenceIdx = roundText.indexOf(fenceMarker);
@@ -1852,7 +2004,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 } else if (hasUnclosedThink && isThinking) {
                   if (_liveThinkInner) {
                     // Extract raw thinking text (strip known thinking wrappers and prefixes)
-                    var thinkText = markdownModule.normalizeThinkingMarkup(roundText)
+                    var thinkText = markdownModule.normalizeThinkingMarkup(_streamDisplayText(roundText))
                       .replace(/<\/?(?:think(?:ing)?|thought)(?:\s+[^>]*)?>/gi, '')
                       .replace(/<\|channel>thought\s*\n?/gi, '')
                       .replace(/<\|channel>response\s*\n?/gi, '')
@@ -2285,6 +2437,14 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 if (!_isBg) {
                   uiModule.showToast('Context compacted — older messages summarized');
                 }
+              } else if (json.type === 'context_trimmed') {
+                if (!_isBg) {
+                  const d = json.data || {};
+                  const before = Number(d.messages_before || 0);
+                  const after = Number(d.messages_after || 0);
+                  const detail = before && after && before > after ? ` (${after}/${before} messages sent)` : '';
+                  uiModule.showToast(`Context trimmed for this model${detail}`);
+                }
               } else if (json.type === 'metrics') {
                 metrics = json.data;
                 if (!_isBg && holder && metrics) {
@@ -2331,7 +2491,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 if (!roundFinalized) {
                   roundFinalized = true;
                   if (spinner && spinner.element) spinner.destroy();
-                  const dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(_stripDocumentFenceForChat(roundText)));
+                  const dt = markdownModule.normalizeThinkingMarkup(_streamDisplayText(roundText));
                   if (dt.trim()) {
                     var _body3 = roundHolder.querySelector('.body');
                     var _contentEl3 = _ensureStreamLayout(_body3);
@@ -2555,6 +2715,25 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 if (json.ui_event) {
                   chatStream.handleUIControl(json);
                 }
+                // Native document tool calls can arrive as a completed
+                // tool_output without the text-fence streaming path. Open the
+                // document editor from the real doc metadata carried on the
+                // tool result so "create a document" never leaves only a chat
+                // link behind if the later doc_update event is missed.
+                if (
+                  documentModule
+                  && json.doc_id
+                  && ['create_document', 'update_document', 'edit_document'].includes(json.tool)
+                ) {
+                  documentModule.handleDocUpdate({
+                    type: 'doc_update',
+                    doc_id: json.doc_id,
+                    title: json.document_title || '',
+                    language: json.document_language || '',
+                    version: json.document_version || 1,
+                    content: json.document_content || '',
+                  });
+                }
 
                 // Schedule a thinking spinner between tool rounds (short delay so
                 // agent_step in the same SSE chunk can cancel it before it shows)
@@ -2673,6 +2852,22 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 const chatBox = document.getElementById('chat-history');
                 chatBox.appendChild(budgetDiv);
 
+              } else if (json.type === 'loop_breaker_triggered' || json.type === 'intent_nudge_exhausted') {
+                if (_isBg) continue;
+                _cancelThinkingTimer();
+                _removeThinkingSpinner();
+                const guardDiv = document.createElement('div');
+                guardDiv.className = 'stopped-indicator';
+                const guardLabel = document.createElement('span');
+                guardLabel.textContent = `[Agent guard: ${json.message || json.reason || 'internal stop'}]`;
+                guardDiv.appendChild(guardLabel);
+                const targetBody = roundHolder && roundHolder.querySelector('.body');
+                if (targetBody) targetBody.appendChild(guardDiv);
+                else {
+                  const chatBox = document.getElementById('chat-history');
+                  if (chatBox) chatBox.appendChild(guardDiv);
+                }
+
               } else if (json.type === 'teacher_takeover') {
                 if (_isBg) continue;
                 _cancelThinkingTimer();
@@ -2807,7 +3002,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         }
 
         // Finalize the last round's bubble — flatten stream-content wrapper for clean DOM
-        const finalDisplay = stripToolBlocks(_stripDocumentFenceForChat(roundText, { final: _docFenceOpened }));
+        const finalDisplay = _streamDisplayText(roundText, { final: _docFenceOpened });
         if (finalDisplay.trim()) {
           var _body4 = roundHolder.querySelector('.body');
           // Preserve sources expanded state before final render
@@ -2873,11 +3068,11 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
           _body4b.innerHTML = _sourcesData ? _buildSourcesBox(_sourcesData, _sourcesType, _wasExpanded2) : _sourcesHtml;
         } else if (roundHolder !== holder) {
           // Check if there's thinking content worth showing
-          const _thinkingOnly = markdownModule.extractThinkingBlocks(roundText);
+          const _thinkingOnly = markdownModule.extractThinkingBlocks(_streamDisplayText(roundText));
           if (_thinkingOnly.thinkingBlocks?.length && !_thinkingOnly.content) {
             // Show thinking in a collapsed section even if no visible reply text
             const _body4c = roundHolder.querySelector('.body');
-            if (_body4c) _body4c.innerHTML = markdownModule.processWithThinking(roundText);
+            if (_body4c) _body4c.innerHTML = markdownModule.processWithThinking(_streamDisplayText(roundText));
           } else {
             roundHolder.style.display = 'none';
             // Thread above expected a bubble below — remove has-bottom since bubble is hidden
@@ -3092,6 +3287,21 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
             return;
           }
 
+          if (abortReason === 'stale-local') {
+            const staleMsg = 'Stream connection ended. Composer unlocked; send again if needed.';
+            if (holder && !accumulated) {
+              holder.querySelector('.body').innerHTML =
+                `<div style="opacity:0.7;font-style:italic;padding:4px 0;">[${staleMsg}]</div>`;
+            } else if (holder && accumulated) {
+              const staleNote = document.createElement('div');
+              staleNote.className = 'stopped-indicator';
+              staleNote.innerHTML = `<span style="opacity:0.7;">[${staleMsg}]</span>`;
+              holder.querySelector('.body').appendChild(staleNote);
+            }
+            currentAbort = null;
+            return;
+          }
+
           // User-initiated stop (or browser navigation abort).
           // Stopped before any text arrived — keep the bubble as a
           // "Cancelled by user" record (so it survives a refresh).
@@ -3398,12 +3608,51 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     box.appendChild(bar);
     if (uiModule.scrollHistory) uiModule.scrollHistory();
   }
+  async function _probeStaleLocalStream() {
+    if (!isStreaming || _staleStreamProbeInFlight) return;
+    if (Date.now() - _lastReaderActivity < STALE_LOCAL_STREAM_MS) return;
+    const sid = _streamSessionId || (sessionModule.getCurrentSessionId && sessionModule.getCurrentSessionId());
+    if (!sid) return;
+    if (_backgroundStreams.has(sid) || (sessionModule.getCurrentSessionId && sessionModule.getCurrentSessionId() !== sid)) return;
+    _staleStreamProbeInFlight = true;
+    try {
+      const res = await fetch(`${API_BASE}/api/chat/stream_status/${encodeURIComponent(sid)}`, {
+        credentials: 'same-origin',
+        cache: 'no-store',
+      });
+      if (!isStreaming || _backgroundStreams.has(sid)) return;
+      if (res.status !== 404) return;
+
+      console.warn('[stream-watchdog] Local stream was stale and server has no active stream. Unlocking composer.');
+      if (currentAbort && !currentAbort.signal.aborted) {
+        currentAbort._reason = 'stale-local';
+        currentAbort.abort();
+      }
+      isStreaming = false;
+      _setForegroundChatBusy(false);
+      _sendInFlight = false;
+      if (_webLockRelease) {
+        _webLockRelease();
+        _webLockRelease = null;
+      }
+      const submitBtn = document.querySelector('.send-btn');
+      if (submitBtn) updateSubmitButton('idle', submitBtn);
+      const messageInput = uiModule.el('message');
+      if (messageInput) messageInput.disabled = false;
+      _drainQueuedAgentRequests();
+    } catch (err) {
+      console.warn('[stream-watchdog] Stream status probe failed:', err);
+    } finally {
+      _staleStreamProbeInFlight = false;
+    }
+  }
+
   function _startStallWatchdog() {
-    // Disabled: the server-side stall detector / auto-continue (agent
+    // Keep the old noisy stall banner disabled. This watchdog only unlocks
-    // loop-breaker) handles quiet/stalled streams now, so the manual
+    // a dead local stream after the backend confirms no active stream exists.
-    // "Quiet for Nm — still working?" banner is redundant (and annoying).
     if (_stallWatchdog) { clearInterval(_stallWatchdog); _stallWatchdog = null; }
     _removeStallBanner();
+    _stallWatchdog = setInterval(_probeStaleLocalStream, 5000);
   }
   function _stopStallWatchdog() {
     if (_stallWatchdog) { clearInterval(_stallWatchdog); _stallWatchdog = null; }
@@ -3565,7 +3814,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     };
 
     const renderDelta = () => {
-      const dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(_stripDocumentFenceForChat(roundText, { final: docFenceOpened })));
+      const dt = markdownModule.normalizeThinkingMarkup(_streamDisplayText(roundText, { final: docFenceOpened }));
       if (docFenceOpened && !dt.trim()) {
         _showDocumentWritingStatus(contentDiv);
       } else {

static/js/chatRenderer.js

@@ -474,6 +474,10 @@ const XML_INVOKE_RE = /<invoke\s+name=['"][^'"]*['"]>[\s\S]*?<\/invoke>/gi;
 // (e.g. mid-stream before the closing tag arrives).
 const DSML_TOOL_RE = /<\s*[｜|]+\s*DSML\s*[｜|]+\s*tool_calls\s*>[\s\S]*?(?:<\s*\/\s*[｜|]+\s*DSML\s*[｜|]+\s*tool_calls\s*>|$)/gi;
 const DSML_STRAY_RE = /<\s*\/?\s*[｜|]+\s*DSML\s*[｜|]+[^>]*>/gi;
+const DSML_INVOKE_RE = /<\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\s*>|$)/gi;
+const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi;
+const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi;
+const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\|?end\|?|\/?\|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
 // Self-narration about tool results (model echoing stdout/exit_code)
 const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi;
 
@@ -911,9 +915,13 @@ export function stripToolBlocks(text) {
   let cleaned = text.replace(TOOL_CALL_RE, '');
   if (EXEC_FENCE_RE) cleaned = cleaned.replace(EXEC_FENCE_RE, stripExecutedFence);
   cleaned = cleaned.replace(DSML_TOOL_RE, '');
+  cleaned = cleaned.replace(DSML_INVOKE_RE, '');
   cleaned = cleaned.replace(DSML_STRAY_RE, '');
   cleaned = cleaned.replace(XML_TOOL_CALL_RE, '');
   cleaned = cleaned.replace(XML_INVOKE_RE, '');
+  cleaned = cleaned.replace(RAW_OPENAI_TOOL_JSON_RE, '');
+  cleaned = cleaned.replace(QWEN_ROLE_MARKER_RE, '');
+  cleaned = cleaned.replace(QWEN_BARE_MARKER_RE, ' ');
   cleaned = cleaned.replace(TOOL_NARRATION_RE, '');
   cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
   return cleaned.trim();
@@ -1152,17 +1160,41 @@ document.addEventListener('click', function(e) {
   while (_t && _t.nodeType === Node.TEXT_NODE) _t = _t.parentElement;
   const a = _t && _t.closest && _t.closest('a[href]');
   if (!a) return;
-  const href = a.getAttribute('href') || '';
+  const rawHref = a.getAttribute('href') || '';
+  let href = rawHref;
+  try {
+    const parsed = new URL(rawHref, window.location.origin);
+    if (parsed.origin === window.location.origin && parsed.pathname === window.location.pathname) {
+      href = parsed.hash || rawHref;
+    }
+  } catch (_) {}
   if (!href.startsWith('#')) return;
-  const m = href.match(/^#(session|document|note|image|email|event|task|skill|research)-(.+)$/);
+  let m = href.match(/^#(session|document|note|image|email|event|task|skill|research)-(.+)$/);
+  if (!m) {
+    const noteOpen = href.match(/^#open=notes&note=([^&]+)/);
+    if (noteOpen) m = ['note', 'note', decodeURIComponent(noteOpen[1])];
+  }
+  if (!m) {
+    const bareSession = href.match(/^#([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i);
+    if (bareSession) m = ['session', 'session', bareSession[1]];
+  }
   if (!m) return;
   e.preventDefault();
   e.stopPropagation();
   const [, kind, id] = m;
   if (kind === 'session') {
+    try {
+      a.classList.add('is-loading');
+      a.setAttribute('aria-busy', 'true');
+    } catch {}
     import('./sessions.js').then(mod => {
       const fn = mod.selectSession || (mod.default && mod.default.selectSession);
-      if (fn) fn(id);
+      if (fn) return fn(id, { showLoading: true, immediateLoading: true });
+    }).finally(() => {
+      try {
+        a.classList.remove('is-loading');
+        a.removeAttribute('aria-busy');
+      } catch {}
     });
   } else if (kind === 'document') {
     import('./document.js').then(mod => {
@@ -1175,6 +1207,11 @@ document.addEventListener('click', function(e) {
     import('./notes.js').then(mod => {
       const open = mod.openNote || (mod.default && mod.default.openNote);
       if (open) open(id);
+      try {
+        if (/^#(?:note-|open=notes&note=)/.test(window.location.hash || '')) {
+          history.replaceState(null, '', window.location.pathname + window.location.search);
+        }
+      } catch (_) {}
     }).catch(() => {});
   } else if (kind === 'image') {
     import('./gallery.js').then(mod => {
@@ -1208,7 +1245,7 @@ document.addEventListener('click', function(e) {
       if (open) open(id);
     }).catch(() => {});
   }
-});
+}, true);
 
 /**
  * Build a generated-image bubble element.
@@ -1326,6 +1363,25 @@ export function buildImageBubble(imageUrl, prompt, model, size, quality, imageId
   });
   actions.appendChild(editBtn);
 
+  if (imageId) {
+    const galleryBtn = document.createElement('button');
+    galleryBtn.className = 'footer-copy-btn footer-open-gallery-btn';
+    galleryBtn.type = 'button';
+    galleryBtn.title = 'Open in gallery';
+    galleryBtn.innerHTML = '<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2"/><circle cx="8.5" cy="8.5" r="1.5"/><path d="M21 15l-5-5L5 21"/></svg><span>Open in gallery</span>';
+    galleryBtn.addEventListener('click', async (e) => {
+      e.stopPropagation();
+      try {
+        const mod = await import('./gallery.js');
+        const open = mod.openGalleryImage || (mod.default && mod.default.openGalleryImage);
+        if (open) open(imageId);
+      } catch (err) {
+        console.error('[chat] open in gallery failed', err);
+      }
+    });
+    actions.appendChild(galleryBtn);
+  }
+
   const delBtn = document.createElement('button');
   delBtn.className = 'footer-copy-btn footer-delete-btn';
   delBtn.type = 'button';
@@ -1789,19 +1845,16 @@ export function displayMetrics(messageElement, metrics) {
     }
   }
 
-  // Default: show tok/s if available, else fall back to other stats
+  // Keep token counts in the Message Stats popup; the footer should stay slim.
   const costStr0 = cost !== null ? `$${cost < 0.01 ? cost.toFixed(4) : cost.toFixed(3)}` : null;
-  const metricsLabel = tps != null && tps !== 'undefined'
+  const hasTps = tps != null && tps !== 'undefined';
+  const metricsLabel = hasTps
     ? `${tps} tok/s`
     : costStr0
-      ? `${outputTokens} tok · ${costStr0}`
+      ? costStr0
-      : outputTokens
+      : responseTime != null
-        ? `${outputTokens} tok · ${responseTime != null ? responseTime + 's' : ''}`
+        ? `${responseTime}s`
-        : inputTokens
+        : '';
-          ? `${inputTokens} in${responseTime != null ? ' · ' + responseTime + 's' : ''}`
-          : responseTime != null
-            ? `${responseTime}s`
-            : '';
   if (!metricsLabel) return;
   metricsContainer.textContent = metricsLabel;
   metricsContainer.style.cursor = 'pointer';
@@ -2442,8 +2495,8 @@ export function addMessage(role, content, modelName, metadata) {
         .trim();
     }
 
-    wrap.dataset.raw = text;
+	    wrap.dataset.raw = text;
-    if (metadata?._db_id) wrap.dataset.dbId = metadata._db_id;
+	    if (metadata?._db_id) wrap.dataset.dbId = metadata._db_id;
     // Prepend sources box if saved in metadata
     var sourcesPrefix = '';
     var findingsSuffix = '';
@@ -2466,9 +2519,10 @@ export function addMessage(role, content, modelName, metadata) {
         '<think' + (thinkTime ? ` time="${thinkTime}"` : '') + '>' + metadata.thinking + '</think>\n\n' + text
       );
       b.innerHTML = sourcesPrefix + thinkHtml + findingsSuffix;
-    } else {
+	    } else {
-      b.innerHTML = sourcesPrefix + markdownModule.processWithThinking(text) + findingsSuffix;
+	      b.innerHTML = sourcesPrefix + markdownModule.processWithThinking(text) + findingsSuffix;
-    }
+	    }
+	    b.dataset.raw = text;
 
     // The vision/OCR caption is stripped from the displayed text above (so the
     // bubble doesn't show the raw model output) but no longer rendered as an

static/js/chatStream.js

@@ -185,9 +185,17 @@ export function handleUIControl(uiData) {
 
     } else if (uiEvent === 'open_email_reply' || uiData.ui_event === 'open_email_reply') {
       try {
-        var existingDocId = documentModule && documentModule.findEmailDocId
+        var activeCtx = documentModule && documentModule.getActiveEmailComposerContext
-          ? documentModule.findEmailDocId(uiData.uid, uiData.folder || 'INBOX')
+          ? documentModule.getActiveEmailComposerContext()
           : null;
+        var sameActiveDraft = activeCtx
+          && String(activeCtx.sourceUid || '') === String(uiData.uid || '')
+          && String(activeCtx.sourceFolder || 'INBOX') === String(uiData.folder || 'INBOX');
+        var existingDocId = sameActiveDraft && activeCtx.docId
+          ? activeCtx.docId
+          : (documentModule && documentModule.findEmailDocId
+            ? documentModule.findEmailDocId(uiData.uid, uiData.folder || 'INBOX')
+            : null);
         if (existingDocId && documentModule.replaceEmailReplyBody) {
           if (documentModule.loadDocument) documentModule.loadDocument(existingDocId);
           documentModule.replaceEmailReplyBody(existingDocId, uiData.body || '', { force: true });

static/js/cookbook-deps-recipes.js

@@ -49,6 +49,16 @@ const _RECIPES = [
     },
   },
 
+  // ── MLX ───────────────────────────────────────────────────────────────
+  {
+    backend: 'mlx_lm',
+    label: 'Any MLX model',
+    match: () => true,
+    variants: {
+      pip:    { commands: ['uv pip install -U mlx-lm'] },
+    },
+  },
+
   // ── llama.cpp ─────────────────────────────────────────────────────────
   {
     backend: 'llama_cpp',
@@ -75,7 +85,7 @@ export function recipeCommands(recipe, variant) {
 // Backends we surface a recipe panel for. Other rows in the Dependencies
 // list keep the existing flat Install/Reinstall button without an expand
 // affordance.
-export const RECIPE_BACKENDS = new Set(['vllm', 'sglang', 'llama_cpp']);
+export const RECIPE_BACKENDS = new Set(['vllm', 'sglang', 'mlx_lm', 'llama_cpp']);
 
 // All recipe entries for a given backend, in catalog order. The first one
 // is the model-specific match (when present); the last is always the

static/js/cookbook-diagnosis.js

@@ -149,6 +149,118 @@ function _openCpuServeEdit(panel) {
   });
 }
 
+function _taskForDiagnosisPanel(panel) {
+  const taskEl = panel?.closest?.('.cookbook-task');
+  const taskId = taskEl?.dataset?.taskId || '';
+  if (!taskId) return null;
+  return (_loadTasks() || []).find(t => t.sessionId === taskId) || null;
+}
+
+function _pythonFromServeCmd(cmd) {
+  const s = String(cmd || '');
+  const abs = s.match(/(?:^|\s)(\/[^\s]+\/bin\/python3?)(?=\s+-m\s+(?:sglang\.launch_server|mlx_lm\.server))/);
+  if (abs) return abs[1];
+  const rel = s.match(/(?:^|\s)(python3?)(?=\s+-m\s+(?:sglang\.launch_server|mlx_lm\.server))/);
+  return rel ? rel[1] : '';
+}
+
+function _pythonForDiagnosisPanel(panel) {
+  const task = _taskForDiagnosisPanel(panel);
+  const fromCmd = _pythonFromServeCmd(task?.payload?._cmd || '');
+  if (fromCmd) return fromCmd;
+  return (_envState.env === 'venv' && _envState.envPath)
+    ? `${_envState.envPath.replace(/\/+$/, '')}/bin/python3`
+    : 'python3';
+}
+
+function _sglangKernelRepairCommand(panel) {
+  return `${_pythonForDiagnosisPanel(panel)} -m pip install -U --force-reinstall --no-cache-dir sglang-kernel`;
+}
+
+function _mlxLmInstallCommand(panel) {
+  return `${_pythonForDiagnosisPanel(panel)} -m pip install -U mlx-lm`;
+}
+
+async function _repairSglangKernel(panel) {
+  const task = _taskForDiagnosisPanel(panel);
+  uiModule.showToast('Repairing sglang-kernel on the selected server...');
+  await _launchServeTask(
+    'repair-sglang-kernel',
+    'pip-update',
+    _sglangKernelRepairCommand(panel),
+    null,
+    task?.remoteHost || undefined,
+    task ? {
+      serverKey: task.remoteServerKey || task.remoteHost || '',
+      serverName: task.remoteServerName || task.remoteHost || '',
+    } : null,
+  );
+}
+
+async function _installMlxLm(panel) {
+  const task = _taskForDiagnosisPanel(panel);
+  uiModule.showToast('Installing MLX LM on the selected server...');
+  await _launchServeTask(
+    'install-mlx-lm',
+    'pip-update',
+    _mlxLmInstallCommand(panel),
+    null,
+    task?.remoteHost || undefined,
+    _diagnosisTargetMeta(task),
+  );
+}
+
+function _diagnosisTargetMeta(task) {
+  return task ? {
+    serverKey: task.remoteServerKey || task.remoteHost || '',
+    serverName: task.remoteServerName || task.remoteHost || '',
+  } : null;
+}
+
+function _gpuCleanupCommand() {
+  return `set -u
+echo "[odysseus] Clearing GPU compute processes..."
+if command -v nvidia-smi >/dev/null 2>&1; then
+  pids="$(nvidia-smi --query-compute-apps=pid --format=csv,noheader,nounits 2>/dev/null | tr -d " " | grep -E "^[0-9]+$" | sort -u)"
+  if [ -z "$pids" ]; then
+    echo "[odysseus] No NVIDIA compute processes found."
+    exit 0
+  fi
+  echo "[odysseus] GPU PIDs: $pids"
+  ps -fp $pids 2>/dev/null || true
+  echo "[odysseus] Sending TERM..."
+  kill -TERM $pids || true
+  sleep 3
+  alive=""
+  for pid in $pids; do
+    if kill -0 "$pid" 2>/dev/null; then alive="$alive $pid"; fi
+  done
+  if [ -n "$alive" ]; then
+    echo "[odysseus] Force killing remaining GPU PIDs:$alive"
+    kill -KILL $alive || true
+  fi
+  sleep 1
+  remaining="$(nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader,nounits 2>/dev/null | sed "/^$/d" || true)"
+  if [ -n "$remaining" ]; then
+    echo "[odysseus] GPU processes still remain:"
+    echo "$remaining"
+    exit 2
+  fi
+  echo "[odysseus] GPU cleanup complete. No NVIDIA compute processes remain."
+else
+  echo "[odysseus] nvidia-smi not found; falling back to common model-server process cleanup."
+  pkill -TERM -f "sglang.launch_server|vllm|llama-server|text-generation-launcher|aphrodite" || true
+  sleep 3
+  pkill -KILL -f "sglang.launch_server|vllm|llama-server|text-generation-launcher|aphrodite" || true
+  echo "[odysseus] Fallback cleanup complete."
+fi`;
+}
+
+async function _clearGpuProcesses(panel) {
+  uiModule.showToast('Clearing GPU compute processes on the selected server...');
+  await _runQuickCmd(panel, _gpuCleanupCommand());
+}
+
 // Infer the gated base repo that single-file checkpoints need configs from
 function _inferBaseRepo(text) {
   if (!text) return null;
@@ -161,6 +273,25 @@ function _inferBaseRepo(text) {
 }
 
 export const ERROR_PATTERNS = [
+  {
+    pattern: /tmux is required|tmux.*not found|tmux:\s*command not found|command not found:\s*tmux|No such file or directory:\s*['"]?tmux/i,
+    message: 'tmux is missing on this server.',
+    suggestion: 'Suggested action: open Dependencies and install tmux on the selected server.',
+    fixes: [
+      { label: 'Open tmux dependency', action: () => _openCookbookDependencies('tmux') },
+      { label: 'Copy apt install', action: () => _copyText('sudo apt install -y tmux') },
+      { label: 'Copy pacman install', action: () => _copyText('sudo pacman -S --needed tmux') },
+    ],
+  },
+  {
+    pattern: /Port \d+ is already serving|port is occupied by a different model|choose another port before launching/i,
+    message: 'Serve port is already occupied by another model.',
+    suggestion: 'Suggested action: stop the old server or choose a different port before relaunching.',
+    fixes: [
+      { label: 'Edit serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+      { label: 'Copy check command', action: () => _copyText('curl http://127.0.0.1:PORT/v1/models') },
+    ],
+  },
   {
     pattern: /No available memory for the cache blocks|Available KV cache memory:.*-/i,
     message: 'No GPU memory left for KV cache after loading model.',
@@ -179,6 +310,39 @@ export const ERROR_PATTERNS = [
       { label: 'Retry with --max-num-seqs 32', action: (panel) => _serveAutoRetry(panel, '--max-num-seqs 32') },
     ],
   },
+  {
+    pattern: /Loaded weights leave no GPU memory for the KV cache under --mem-fraction-static|Raise --mem-fraction-static above/i,
+    message: 'SGLang static memory fraction is too low for the loaded weights.',
+    suggestion: 'Suggested action: retry with --mem-fraction-static 0.80 so weights fit and KV cache can still allocate.',
+    fixes: [
+      { label: 'Retry mem 0.80', action: (panel) => _serveAutoRetryReplace(panel, '--mem-fraction-static', '0.80') },
+      { label: 'Retry mem 0.82', action: (panel) => _serveAutoRetryReplace(panel, '--mem-fraction-static', '0.82') },
+      { label: 'Edit serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+    ],
+  },
+  {
+    pattern: /get_paged_mqa_logits_metadata|deepseek_v4_backend\.py|paged_mqa_metadata\.cuh:113.*CUDA error:\s*invalid argument/i,
+    message: 'SGLang DeepSeek-V4 attention metadata kernel failed on this GPU/runtime.',
+    suggestion: 'Suggested action: stop retrying graph/memory tweaks for this exact FP8 command. SGLang’s RTX PRO 6000 recipe uses the original deepseek-ai/DeepSeek-V4-Flash checkpoint with --moe-runner-backend marlin, not the converted sgl-project FP8 checkpoint. Try that recipe/checkpoint, official SGLang container/nightly, or supported Hopper/Blackwell hardware.',
+    fixes: [
+      { label: 'Edit serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+      { label: 'Copy error', action: (panel) => {
+        const task = panel.closest('.cookbook-task');
+        const text = task?.querySelector('.cookbook-task-output')?.textContent || task?.textContent || '';
+        _copyText(text.trim());
+      } },
+    ],
+  },
+  {
+    pattern: /Capture cuda graph failed|cuda graph failed|paged_mqa_metadata|cuda-graph-backend-decode|cuda-graph-max-bs-decode|CUDA error:\s*invalid argument/i,
+    message: 'SGLang failed while capturing decode CUDA graphs.',
+    suggestion: 'Suggested action: disable SGLang decode CUDA graph for this launch. DeepSeek-V4 is reaching graph capture, but this kernel is failing on the target hardware.',
+    fixes: [
+      { label: 'Disable decode graph', action: (panel) => _serveAutoRetryReplace(panel, '--cuda-graph-backend-decode', 'disabled') },
+      { label: 'Retry mem 0.80', action: (panel) => _serveAutoRetryReplace(panel, '--mem-fraction-static', '0.80') },
+      { label: 'Edit serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+    ],
+  },
   {
     pattern: /CUDA out of memory|torch\.cuda\.OutOfMemoryError|CUDA error: out of memory/i,
     message: 'GPU ran out of memory. Try more GPUs (higher TP) or lower context.',
@@ -334,6 +498,18 @@ export const ERROR_PATTERNS = [
       { label: 'Enable enforce eager', action: (panel) => _setPanelCheckbox(panel, 'enforce_eager', true) },
     ],
   },
+  {
+    pattern: /memory capacity is unbalanced|Some GPUs may be occupied by other processes|pre_model_load_memory=.*local_gpu_memory/i,
+    message: 'SGLang refused to start because free GPU memory is uneven across the selected tensor-parallel GPUs.',
+    suggestion: 'Suggested action: run Clear GPUs, then relaunch. If it still fails, choose only equally free GPUs or lower TP/context.',
+    fixes: [
+      { label: 'Clear GPUs', action: (panel) => _clearGpuProcesses(panel) },
+      { label: 'Copy clear command', action: () => _copyText(_gpuCleanupCommand()) },
+      { label: 'Edit serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+      { label: 'Set TP to 1', action: (panel) => _setPanelField(panel, 'tp', '1') },
+      { label: 'Lower context', action: (panel) => _setPanelField(panel, 'ctx', '32768') },
+    ],
+  },
   {
     pattern: /KV cache.*too (small|large)|max_model_len.*exceeds|maximum.*context/i,
     message: 'Context length too large for available GPU memory.',
@@ -355,11 +531,14 @@ export const ERROR_PATTERNS = [
     ],
   },
   {
-    pattern: /sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|Please ensure sgl_kernel is properly installed/i,
+    pattern: /sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|(?:Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|Could not load any common_ops library|Please ensure sgl_kernel is properly installed/i,
-    message: 'SGLang native dependencies are missing on this server.',
+    message: 'SGLang native kernel/runtime is missing or mismatched on this server.',
+    suggestion: 'Suggested action: relaunch with Odysseus’ venv CUDA library path fix. If the venv does not contain the matching NVIDIA runtime libs, run Repair sglang-kernel.',
     fixes: [
+      { label: 'Edit / relaunch serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+      { label: 'Repair sglang-kernel', action: (panel) => _repairSglangKernel(panel) },
+      { label: 'Copy repair command', action: (panel) => _copyText(_sglangKernelRepairCommand(panel)) },
       { label: 'Copy OS package command', action: () => _copyText('sudo apt-get install -y libnuma-dev python3.12-dev build-essential') },
-      { label: 'Copy kernel upgrade', action: () => _copyText('python3 -m pip install --upgrade sglang-kernel') },
       { label: 'Open Dependencies', action: () => _openCookbookDependencies('sglang') },
     ],
   },
@@ -371,6 +550,30 @@ export const ERROR_PATTERNS = [
       { label: 'Copy install command', action: () => _copyText('python3 -m pip install "sglang[all]"') },
     ],
   },
+  {
+    pattern: /No module named ['"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed/i,
+    message: 'MLX LM is not installed on this server.',
+    suggestion: 'Suggested action: install mlx-lm in the selected Python environment. MLX serving is intended for Apple Silicon Macs.',
+    fixes: [
+      { label: 'Install MLX LM', action: (panel) => _installMlxLm(panel) },
+      { label: 'Open Dependencies', action: () => _openCookbookDependencies('mlx_lm') },
+      { label: 'Copy install command', action: () => _copyText('python3 -m pip install -U mlx-lm') },
+    ],
+  },
+  {
+    pattern: /Unable to quantize model of type <class ['"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['"]>|QuantizedSwitchLinear/i,
+    message: 'MLX-LM tried to quantize an already-quantized DeepSeek switch layer.',
+    suggestion: 'Suggested action: relaunch from the cached local snapshot path. Odysseus now rewrites MLX repo-id launches to the newest local Hugging Face snapshot when it exists on the selected Mac.',
+    fixes: [
+      { label: 'Edit / relaunch serve', action: (panel) => _openServeEditFromDiagnosis(panel) },
+      { label: 'Open Dependencies', action: () => _openCookbookDependencies('mlx_lm') },
+      { label: 'Copy error', action: (panel) => {
+        const task = panel.closest('.cookbook-task');
+        const text = task?.querySelector('.cookbook-task-output')?.textContent || task?.textContent || '';
+        _copyText(text.trim());
+      } },
+    ],
+  },
   {
     pattern: /No accelerator \(CUDA, XPU, HPU, NPU, MUSA, MPS\) is available|Triton is not supported on current platform/i,
     message: 'SGLang needs a visible GPU/accelerator on this server.',
@@ -834,22 +1037,38 @@ export function _clearDiagnosis(panel) {
 // ── Quick command ──
 
 export async function _runQuickCmd(panel, cmd) {
+  const task = _taskForDiagnosisPanel(panel);
   let fullCmd = cmd;
-  if (_envState.remoteHost) {
+  const host = task?.remoteHost || _envState.remoteHost || '';
-    fullCmd = _sshCmd(_envState.remoteHost, cmd);
+  const port = task?.sshPort || task?.payload?.ssh_port || _envState.sshPort || '';
+  if (host) {
+    fullCmd = _sshCmd(host, cmd, port);
   }
   const diag = panel.querySelector('.cookbook-diagnosis');
-  if (diag) { diag.classList.remove('hidden'); diag.textContent = `Running: ${fullCmd}...`; }
+  if (diag) {
+    diag.classList.remove('hidden');
+    diag.innerHTML = '<div class="cookbook-diag-message">Running command...</div>';
+  }
 
   try {
-    const res = await fetch('/api/shell/stream', {
+    const res = await fetch('/api/shell/exec', {
       method: 'POST',
       credentials: 'same-origin',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ command: fullCmd }),
+      body: JSON.stringify({ command: fullCmd, timeout: 60 }),
     });
-    if (diag) diag.textContent = res.ok ? `Done: ${cmd}` : `Failed (HTTP ${res.status})`;
+    const data = await res.json().catch(() => ({}));
+    const out = [data.stdout, data.stderr].filter(Boolean).join('\n').trim();
+    const ok = res.ok && Number(data.exit_code ?? 1) === 0;
+    if (diag) {
+      diag.innerHTML = ''
+        + `<div class="cookbook-diag-message">${ok ? 'Command completed.' : 'Command failed.'}</div>`
+        + `<div class="cookbook-diag-suggestion" style="opacity:0.75;margin-top:1px;">Exit code: ${_diagEsc(data.exit_code ?? 'unknown')}</div>`
+        + (out ? `<pre class="cookbook-diag-output" style="margin:6px 0 0;white-space:pre-wrap;max-height:180px;overflow:auto;font-size:10px;line-height:1.35;">${_diagEsc(out)}</pre>` : '');
+    }
   } catch (e) {
-    if (diag) diag.textContent = `Error: ${e.message}`;
+    if (diag) {
+      diag.innerHTML = `<div class="cookbook-diag-message">Command error.</div><div class="cookbook-diag-suggestion">${_diagEsc(e.message)}</div>`;
+    }
   }
 }

static/js/cookbook-hwfit.js

@@ -24,6 +24,9 @@ import {
   _MODELDIR_CHECK_ON,
   _MODELDIR_CHECK_OFF,
   _serverEntryHtml,
+  _serverDefaultHtml,
+  _applyServerSelectColor,
+  _syncServerSelectColors,
   _copyText,
   // Import cookbook.js WITHOUT a ?v= query — the same plain specifier every other
   // importer uses. A query mismatch loads cookbook.js twice as two separate modules
@@ -34,10 +37,59 @@ import spinnerModule from './spinner.js';
 import { _loadTasks, _tmuxGracefulKill, _nextAvailablePort, _taskPort } from './cookbookRunning.js';
 import { openCookbookDependencies } from './cookbook-diagnosis.js';
 
-// Map a serve-backend code (vllm / sglang / llamacpp) → the package name
+// Map a serve-backend code (vllm / sglang / llamacpp / mlx) → the package name
 // the Dependencies API reports. Used to look up "is this backend installed
 // on the target server" before firing a launch.
-const _BACKEND_PKG = { vllm: 'vllm', sglang: 'sglang', llamacpp: 'llama_cpp' };
+const _BACKEND_PKG = { vllm: 'vllm', sglang: 'sglang', llamacpp: 'llama_cpp', mlx: 'mlx_lm' };
+
+function _normalizeCookbookModelDir(dir) {
+  const d = String(dir || '').replaceAll('\u2715', '').replaceAll('\u2716', '').trim();
+  return /^(home|mnt|media|data|opt|srv|var)\//.test(d) ? `/${d}` : d;
+}
+
+function _wireServerColorPicker(entry) {
+  const wrap = entry.querySelector('.cookbook-srv-color-wrap');
+  const select = entry.querySelector('.cookbook-srv-color');
+  const btn = entry.querySelector('.cookbook-srv-color-btn');
+  const menu = entry.querySelector('.cookbook-srv-color-menu');
+  if (!wrap || !select || !btn || !menu || btn.dataset.bound) return;
+  btn.dataset.bound = '1';
+  const close = () => {
+    menu.classList.add('hidden');
+    btn.setAttribute('aria-expanded', 'false');
+  };
+  const open = () => {
+    document.querySelectorAll('.cookbook-srv-color-menu').forEach(m => {
+      if (m !== menu) m.classList.add('hidden');
+    });
+    menu.classList.remove('hidden');
+    btn.setAttribute('aria-expanded', 'true');
+  };
+  btn.addEventListener('click', (e) => {
+    e.stopPropagation();
+    if (menu.classList.contains('hidden')) open();
+    else close();
+  });
+  menu.querySelectorAll('.cookbook-srv-color-item').forEach(item => {
+    item.addEventListener('click', (e) => {
+      e.stopPropagation();
+      const color = item.dataset.color || '';
+      select.value = color;
+      const label = item.querySelector('span:last-child')?.textContent || 'Auto';
+      const labelEl = btn.querySelector('.cookbook-srv-color-label');
+      if (labelEl) labelEl.textContent = label;
+      const swatch = item.style.getPropertyValue('--swatch-color') || color;
+      if (/^#[0-9a-fA-F]{6}$/.test(swatch.trim())) {
+        entry.style.setProperty('--cookbook-server-color', swatch.trim());
+        wrap.style.setProperty('--cookbook-server-color', swatch.trim());
+      }
+      menu.querySelectorAll('.cookbook-srv-color-item').forEach(b => b.classList.toggle('active', b === item));
+      close();
+      select.dispatchEvent(new Event('change', { bubbles: true }));
+    });
+  });
+  document.addEventListener('click', close);
+}
 
 // Pre-launch: ask the deps API whether the chosen backend is present on
 // the target server. Returns true if it's good to go, false if we should
@@ -80,7 +132,6 @@ export let _cachedModelIds = null; // repo IDs already downloaded
 // after the user has switched servers.
 let _hwfitFetchToken = 0;
 let _dismissedHwChips = new Set();
-let _hwfitAutoScanStarted = new Set();
 // Permanently removed (X-clicked) chips. Separate from _dismissedHwChips
 // so the ranker treats "off" and "removed" the same (both ignore the
 // hardware) but the UI keeps "off" chips visible to toggle back on,
@@ -407,15 +458,12 @@ function _manualDisplaySystem(sys, manual) {
 // Signature of everything that affects the result list, so we never paint a
 // cached list under mismatched filters.
 function _scanSig() {
-  const sortEl = document.getElementById('hwfit-sort');
   const tc = document.getElementById('hwfit-gpu-toggles');
   return JSON.stringify({
     h: _envState.remoteHost || '',
     hk: _currentServerValue(),
     u: document.getElementById('hwfit-usecase')?.value || '',
     s: document.getElementById('hwfit-search')?.value?.trim() || '',
-    o: sortEl?.value || 'newest',
-    r: sortEl?.dataset.reverse === '1' ? 1 : 0,
     q: document.getElementById('hwfit-quant')?.value || '',
     c: _ctxValue(),
     g: (tc && typeof tc._activeCount === 'number') ? String(tc._activeCount) : '',
@@ -534,6 +582,13 @@ function _olParseSize(s) {
 function _ollamaToHwfitRows(libModels, vramAvail, ramAvail) {
   const out = [];
   if (!Array.isArray(libModels)) return out;
+  const _ramFitLevel = (need, budget) => {
+    if (!need || !budget || need > budget) return 'too_tight';
+    const ratio = need / budget;
+    if (ratio <= 0.50) return 'perfect';
+    if (ratio <= 0.78) return 'good';
+    return 'marginal';
+  };
   for (const m of libModels) {
     const sizes = (Array.isArray(m.sizes) && m.sizes.length) ? m.sizes : ['latest'];
     for (const sz of sizes) {
@@ -544,10 +599,10 @@ function _ollamaToHwfitRows(libModels, vramAvail, ramAvail) {
       if (vramGb && vramAvail) {
         if (vramGb <= vramAvail * 0.6) fitLevel = 'perfect';
         else if (vramGb <= vramAvail) fitLevel = 'good';
-        else if (ramAvail && vramGb <= ramAvail) fitLevel = 'marginal';
+        else if (ramAvail && vramGb <= ramAvail) fitLevel = _ramFitLevel(vramGb, ramAvail);
         else fitLevel = 'too_tight';
       } else if (vramGb && ramAvail && vramGb <= ramAvail) {
-        fitLevel = 'marginal';
+        fitLevel = _ramFitLevel(vramGb, ramAvail);
       }
       const tag = `${m.name}:${sz}`;
       const paramsLabel = params
@@ -628,21 +683,18 @@ export async function _hwfitFetch(fresh = false, opts = {}) {
       loadingTitle.textContent = 'No cached scan yet';
       loadingTitle.style.cssText = 'font-size:12px;opacity:0.7;';
       const loadingLbl = document.createElement('div');
-      loadingLbl.textContent = 'Scanning hardware…';
+      loadingLbl.textContent = 'Loading model list…';
       loadingLbl.style.cssText = 'font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;';
       loadingDiv.appendChild(loadingTitle);
       loadingDiv.appendChild(loadingLbl);
       list.innerHTML = '';
       list.appendChild(loadingDiv);
-      if (!_hwfitAutoScanStarted.has(_sig)) {
+      setTimeout(() => {
-        _hwfitAutoScanStarted.add(_sig);
+        if (_tk === _hwfitFetchToken) {
-        setTimeout(() => {
+          _resetGpuToggleState();
-          if (_tk === _hwfitFetchToken) {
+          _hwfitFetch(true, { autoFromEmpty: true });
-            _resetGpuToggleState();
+        }
-            _hwfitFetch(true, { autoFromEmpty: true });
+      }, 60);
-          }
-        }, 60);
-      }
       return;
     }
     if (!canKeepPrevious) {
@@ -653,13 +705,15 @@ export async function _hwfitFetch(fresh = false, opts = {}) {
       loadingDiv.style.flexDirection = 'column';
       loadingDiv.style.gap = '6px';
       loadingDiv.appendChild(wp.element);
-      // Text label like the other cookbook tabs: "Loading…", then if the scan runs
+      // Text label like the other cookbook tabs. Only fresh rescans are hardware
-      // long (remote SSH hardware probe), switch to "Scanning hardware…".
+      // probes; normal refreshes are just model ranking/loading from cached hw.
       const loadingLbl = document.createElement('div');
-      loadingLbl.textContent = 'Loading…';
+      loadingLbl.textContent = fresh ? 'Scanning hardware…' : 'Loading models…';
       loadingLbl.style.cssText = 'text-align:center;opacity:0.5;font-size:11px;';
       loadingDiv.appendChild(loadingLbl);
-      setTimeout(() => { if (loadingLbl.isConnected) loadingLbl.textContent = 'Scanning hardware…'; }, 2000);
+      setTimeout(() => {
+        if (loadingLbl.isConnected) loadingLbl.textContent = fresh ? 'Scanning hardware…' : 'Loading model list…';
+      }, 2000);
       list.innerHTML = '';
       list.appendChild(loadingDiv);
       _hwfitCache = null;   // no instant paint — clear until the fetch returns
@@ -703,10 +757,6 @@ export async function _hwfitFetch(fresh = false, opts = {}) {
         _setLastCacheHost('');
       });
   }
-  if (_paintedFromCache && !forceRevalidate) {
-    try { wp.destroy(); } catch {}
-    return;
-  }
   try {
     const sortBy = document.getElementById('hwfit-sort')?.value || 'newest';
     const quantPref = document.getElementById('hwfit-quant')?.value || '';
@@ -722,7 +772,10 @@ export async function _hwfitFetch(fresh = false, opts = {}) {
     if (!hasManualOrDismissed && toggleContainer && toggleContainer._activeGroup) {
       gpuGroupOverride = String(toggleContainer._activeGroup);
     }
-    const params = new URLSearchParams({ limit: '80', sort: sortBy });
+    // Sorting is a table operation, not a different backend query. Fetch a
+    // broad candidate set once, then sort it client-side so VRAM/Params/etc.
+    // do not appear to "filter out" rows by returning a different top-80 slice.
+    const params = new URLSearchParams({ limit: '2500', sort: 'score' });
     if (fresh) params.set('fresh', '1');   // bypass the hardware-scan cache
     if (search) params.set('search', search);
     if (remoteHost) {
@@ -743,6 +796,9 @@ export async function _hwfitFetch(fresh = false, opts = {}) {
     if (hasManualOrDismissed) params.set('_hw_override_ts', String(Date.now()));
     // Image models use a separate registry/endpoint
     const isImageMode = useCase === 'image_gen';
+    if ((fresh || (_paintedFromCache && !search)) && !isImageMode) {
+      params.set('refresh_catalog', '1'); // update HF-backed dynamic catalogs in the background
+    }
     if (!isImageMode) {
       if (useCase) params.set('use_case', useCase);
       if (quantPref) params.set('quant', quantPref);
@@ -1198,9 +1254,9 @@ function _modeLabel(model) {
 export const _hwfitColumns = [
   { key: 'fit', label: 'Fit',    cls: 'hwfit-fit' },
   { key: 'newest', label: 'Model (latest)',  cls: 'hwfit-name' },
+  { key: 'vram',  label: 'VRAM',   cls: 'hwfit-c-vram' },
   { key: 'params',label: 'Param', cls: 'hwfit-c-params' },
   { key: null,    label: 'Quant',  cls: 'hwfit-c-quant' },
-  { key: 'vram',  label: 'VRAM',   cls: 'hwfit-c-vram' },
   { key: 'context',label: 'Ctx',   cls: 'hwfit-c-ctx' },
   { key: 'speed', label: 'Speed',  cls: 'hwfit-c-speed' },
   { key: 'score', label: 'Score',  cls: 'hwfit-c-score' },
@@ -1301,13 +1357,13 @@ export function _hwfitRenderList(el, models) {
       }
     }
     html += `<span class="hwfit-col hwfit-name">${modelLogo(m.name)}${esc(_short)}${_quantSuffix}${moeBadge}${imgBadge}${dlDot}</span>`;
-    html += `<span class="hwfit-col hwfit-c-params">${esc(pcount)}</span>`;
+    html += `<span class="hwfit-col hwfit-c-vram" title="Estimated loaded footprint for this quant/backend/context, including model weights and KV/runtime allowance.">${vramLabel}</span>`;
+    html += `<span class="hwfit-col hwfit-c-params" title="Original total model parameters, not quantized storage size.">${esc(pcount)}</span>`;
     // Truncate the Quant cell to 9 chars + ellipsis so long tags like
     // "FP4-MoE-Mixed" don't push neighboring columns. Full tag stays in title.
     const _qRaw = m.quant || '?';
     const _qShort = _qRaw.length > 9 ? _qRaw.slice(0, 9) + '…' : _qRaw;
     html += `<span class="hwfit-col hwfit-c-quant" title="${esc(_qRaw)}">${esc(_qShort)}</span>`;
-    html += `<span class="hwfit-col hwfit-c-vram">${vramLabel}</span>`;
     html += `<span class="hwfit-col hwfit-c-ctx">${m.is_image_gen ? '\u2014' : ctx}</span>`;
     html += `<span class="hwfit-col hwfit-c-speed">${m.is_image_gen ? '\u2014' : tps + ' t/s'}</span>`;
     html += `<span class="hwfit-col hwfit-c-score">${score}</span>`;
@@ -1356,14 +1412,13 @@ export function _hwfitRenderList(el, models) {
       if (e.target.closest('[data-fit-dot]')) {
         const on = !e.target.classList.contains('active');
         try { localStorage.setItem('hwfit_fit_only_v1', on ? '1' : '0'); } catch {}
-        // Un-toggling the fit filter (off → showing too-tight rows again) is
+        // Un-toggling the fit filter should still keep the list usable: show
-        // typically because the user wants to see the LARGE models they can't
+        // nearest/smallest VRAM first, not a wall of impossible 7000G rows.
-        // run yet — re-sort by VRAM descending so the biggest surface first.
         if (!on) {
           const sortSel = document.getElementById('hwfit-sort');
           if (sortSel) {
             sortSel.value = 'vram';
-            sortSel.dataset.reverse = '0';   // descending (biggest first)
+            sortSel.dataset.reverse = '1';   // ascending (smallest VRAM first)
           }
         }
         _hwfitCache = null;
@@ -1379,7 +1434,9 @@ export function _hwfitRenderList(el, models) {
         sel.dataset.reverse = sel.dataset.reverse === '1' ? '0' : '1';
       } else {
         sel.value = sortKey;
-        sel.dataset.reverse = '0';
+        // VRAM is most useful as "what fits / closest fit first"; descending
+        // buries qwen/gemma-sized rows below absurd impossible footprints.
+        sel.dataset.reverse = sortKey === 'vram' ? '1' : '0';
       }
       _hwfitFetch();
     });
@@ -1751,6 +1808,9 @@ export function _expandModelRow(row, modelData) {
         cmd += ` --context-length ${maxCtx}`;
         cmd += ` --mem-fraction-static ${gpuUtil}`;
         cmd += ' --trust-remote-code';
+      } else if (runBackend === 'mlx') {
+        const bindHost = host ? '0.0.0.0' : '127.0.0.1';
+        cmd = `python3 -m mlx_lm.server --model ${_shellQuote(modelData.name)} --host ${bindHost} --port ${port}`;
       } else if (runBackend === 'llamacpp') {
         const dir = `"$HOME/.cache/huggingface/hub/models--${modelData.name.replace(/\//g, '--')}/snapshots"`;
         const ggufPath = `$({ find ${dir} -name '*-00001-of-*.gguf' 2>/dev/null | sort; find ${dir} -name '*.gguf' 2>/dev/null | sort; } | head -1)`;
@@ -1868,6 +1928,7 @@ const _HWFIT_ENGINE_GLYPHS = {
   '': '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><line x1="4" y1="6" x2="20" y2="6"></line><line x1="4" y1="12" x2="20" y2="12"></line><line x1="4" y1="18" x2="20" y2="18"></line><circle cx="8" cy="6" r="2" fill="currentColor" stroke="none"></circle><circle cx="16" cy="12" r="2" fill="currentColor" stroke="none"></circle><circle cx="10" cy="18" r="2" fill="currentColor" stroke="none"></circle></svg>',
   vllm: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"></path><path d="M14 4l4 9 3-9"></path></svg>',
   sglang: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;"></span>',
+  mlx: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M4 18V6l4 7 4-7v12"></path><path d="M16 6v12"></path><path d="M20 6v12"></path></svg>',
   llamacpp: '<svg width="14" height="14" viewBox="0 0 600 600" fill="none" aria-hidden="true"><path d="M600 392L504.249 558L504.137 557.929C487.252 584.069 458.193 600 426.864 600H120L240 392H600Z" fill="currentColor"></path><path d="M240 392H0L199.602 46.0254C216.032 17.5463 246.411 0 279.29 0H466.154L240 392Z" fill="currentColor"></path></svg>',
   ollama: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;"></span>',
   diffusers: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"></circle><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"></path></svg>',
@@ -2040,15 +2101,22 @@ export function _hwfitInit() {
     ];
     for (const sel of selectors) {
       if (!sel) continue;
-      const currentVal = sel.value;
+      const currentVal = sel.value || _currentServerValue();
-      let html = `<option value="local">Local</option>`;
+      const localSrv = _envState.servers.find(s => !s.host || String(s.host).toLowerCase() === 'local') || {};
+      const localColor = /^#[0-9a-fA-F]{6}$/.test(String(localSrv.color || '').trim()) ? String(localSrv.color).trim() : '';
+      const localLabel = localSrv.name || 'Local';
+      let html = `<option value="local"${localColor ? ` style="color:${uiModule.esc(localColor)};"` : ''}>${uiModule.esc(localColor ? `● ${localLabel}` : localLabel)}</option>`;
       _envState.servers.forEach((s, i) => {
         if (!s.host) return;
         const label = s.name || s.host || `Server ${i + 1}`;
-        html += `<option value="${i}">${uiModule.esc(label)}</option>`;
+        const color = /^#[0-9a-fA-F]{6}$/.test(String(s.color || '').trim()) ? String(s.color).trim() : '';
+        html += `<option value="${uiModule.esc(_serverKey(s))}"${color ? ` style="color:${uiModule.esc(color)};"` : ''}>${uiModule.esc(color ? `● ${label}` : label)}</option>`;
       });
       sel.innerHTML = html;
       sel.value = currentVal;
+      if (sel.selectedIndex < 0) sel.value = _currentServerValue();
+      if (sel.selectedIndex < 0) sel.value = 'local';
+      _applyServerSelectColor(sel);
     }
   }
 
@@ -2066,13 +2134,15 @@ export function _hwfitInit() {
       const port = row.querySelector('.cookbook-srv-port')?.value.trim() || '';
       const env = row.querySelector('.cookbook-srv-env')?.value || 'none';
       const envPath = row.querySelector('.cookbook-srv-path')?.value.trim() || '';
+      const colorRaw = row.querySelector('.cookbook-srv-color')?.value?.trim() || '';
+      const color = /^#[0-9a-fA-F]{6}$/.test(colorRaw) ? colorRaw : '';
       // Collect model directories from tags. Read the authoritative data-dir
       // attribute, not textContent \u2014 the tag now also holds a download-target
       // icon, and textContent would fold the icon/\u2716 glyph into the path.
       const dirTags = entry.querySelectorAll('.cookbook-modeldir-tag');
       const modelDirs = [];
       dirTags.forEach(tag => {
-        const d = (tag.dataset.dir || '').replaceAll('\u2715', '').replaceAll('\u2716', '').trim();
+        const d = _normalizeCookbookModelDir(tag.dataset.dir || '');
         if (d) modelDirs.push(d);
       });
       if (!modelDirs.length) modelDirs.push('~/.cache/huggingface/hub');
@@ -2080,7 +2150,7 @@ export function _hwfitInit() {
       const dlEl = entry.querySelector('.cookbook-modeldir-dl.active');
       const downloadDir = dlEl ? (dlEl.dataset.dlDir || '') : '';
       const platform = entry.dataset.platform || '';
-      _envState.servers.push({ name, host: host || '', port, env, envPath, modelDirs, modelDir: modelDirs.filter(d => d !== '~/.cache/huggingface/hub')[0] || modelDirs[0], downloadDir, platform });
+      _envState.servers.push({ name, host: host || '', port, env, envPath, color, modelDirs, modelDir: modelDirs.filter(d => d !== '~/.cache/huggingface/hub')[0] || modelDirs[0], downloadDir, platform });
     });
     // Do NOT auto-change the selected host here. _syncServers can run while the
     // servers DOM is mid-render — host fields that are disabled/readonly read as
@@ -2259,8 +2329,7 @@ export function _hwfitInit() {
         document.querySelectorAll('.cookbook-srv-default').forEach(b => {
           const on = !!_envState.defaultServer && b.dataset.srvKey === _envState.defaultServer;
           b.classList.toggle('active', on);
-          // Keep the "default" label after the icon (don't overwrite it).
+          b.innerHTML = _serverDefaultHtml(on);
-          b.innerHTML = (on ? _MODELDIR_CHECK_ON : _MODELDIR_CHECK_OFF) + '<span class="cookbook-srv-default-label">default</span>';
           b.title = on ? 'Default server — Cookbook opens here' : 'Make this the default server';
         });
         // Apply immediately so the dropdowns reflect it without reopening
@@ -2307,10 +2376,28 @@ export function _hwfitInit() {
         uiModule.showToast('SSH setup command copied');
       });
     }
+    _wireServerColorPicker(entry);
     entry.querySelectorAll('input, select').forEach(el => {
       el.addEventListener('change', () => {
         const selectedBefore = _envState.remoteHost || '';
         const entryHost = entry.querySelector('.cookbook-srv-host')?.value?.trim() || '';
+        const color = entry.querySelector('.cookbook-srv-color')?.value?.trim() || '';
+        const hasColor = /^#[0-9a-fA-F]{6}$/.test(color);
+        const colorWrap = entry.querySelector('.cookbook-srv-color-wrap');
+        if (hasColor) {
+          entry.style.setProperty('--cookbook-server-color', color);
+          colorWrap?.style.setProperty('--cookbook-server-color', color);
+        } else {
+          const autoColor = (colorWrap?.style.getPropertyValue('--cookbook-server-color') || entry.style.getPropertyValue('--cookbook-server-color') || '').trim();
+          if (/^#[0-9a-fA-F]{6}$/.test(autoColor)) {
+            entry.style.setProperty('--cookbook-server-color', autoColor);
+            colorWrap?.style.setProperty('--cookbook-server-color', autoColor);
+          } else {
+            entry.style.removeProperty('--cookbook-server-color');
+            colorWrap?.style.removeProperty('--cookbook-server-color');
+          }
+        }
+        colorWrap?.classList.toggle('has-color', true);
         _syncServers();
         _rebuildServerSelect();
         if (selectedBefore && selectedBefore === entryHost) {
@@ -2320,6 +2407,11 @@ export function _hwfitInit() {
         if (!entry.querySelector('.cookbook-server-key-panel')?.classList.contains('hidden')) {
           _populateServerKeyPanel(entry, false);
         }
+        const saveBtn = entry.querySelector('.cookbook-server-save-btn.saved');
+        if (saveBtn) {
+          saveBtn.classList.remove('saved');
+          saveBtn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><polyline points="17 21 17 13 7 13 7 21"/><polyline points="7 3 7 8 15 8"/></svg>Save';
+        }
       });
     });
     // Manual connectivity test after editing host or port. Existing saved
@@ -2341,7 +2433,7 @@ export function _hwfitInit() {
         _hwfitFetch();
       });
     }
-    // Save button on a brand-new server entry: persist + confirm with a check.
+    // Save button: persist + confirm with a check.
     const saveBtn = entry.querySelector('.cookbook-server-save-btn');
     if (saveBtn && !saveBtn.dataset.bound) {
       saveBtn.dataset.bound = '1';
@@ -2359,6 +2451,7 @@ export function _hwfitInit() {
         } catch (_) {}
         saveBtn.classList.add('saved');
         saveBtn.innerHTML = '<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="#50fa7b" stroke-width="2.6" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><polyline points="20 6 9 17 4 12"/></svg>Saved';
+        uiModule.showToast('Server saved');
       });
     }
     const rmBtn = entry.querySelector('.cookbook-server-rm');
@@ -2520,7 +2613,7 @@ export function _hwfitInit() {
       // Build the new entry with the SAME template as existing servers (Model
       // Directory header, default checkmark, platform icon) \u2014 isNew swaps the
       // delete button for a Save button. forceRemote keeps it editable.
-      const blank = { host: '', name: '', port: '', env: 'none', envPath: '', platform: '', modelDirs: ['~/.cache/huggingface/hub'] };
+      const blank = { host: '', name: '', port: '', env: 'none', envPath: '', color: '', platform: '', modelDirs: ['~/.cache/huggingface/hub'] };
       const wrap = document.createElement('div');
       wrap.innerHTML = _serverEntryHtml(blank, idx, _envState.defaultServer || '', true, true);
       const entry = wrap.firstElementChild;
@@ -2554,6 +2647,7 @@ export function _hwfitInit() {
         }
       }
       _persistEnvState();
+      _applyServerSelectColor(serverSelect);
       // Keep the other server dropdowns (Download / Cache / Deps) in sync. The
       // download-input button reads #hwfit-dl-server *directly*, so without this
       // it kept its old value and downloads went to the wrong host even
@@ -2561,6 +2655,7 @@ export function _hwfitInit() {
       document.querySelectorAll('#hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(sel => {
         if (!sel || sel.tagName !== 'SELECT') return;
         sel.value = _currentServerValue();
+        _applyServerSelectColor(sel);
       });
       _hwfitCache = null;
       // Reset GPU-toggle state (no flicker) so the new server's hardware re-renders.
@@ -2568,5 +2663,6 @@ export function _hwfitInit() {
       _hwfitFetch();
     });
   }
+  _syncServerSelectColors();
 
 }

static/js/cookbook.js

@@ -60,6 +60,11 @@ if (typeof window !== 'undefined' && !window._tagScrollGuardWired) {
 export const _MODELDIR_CHECK_OFF = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="9"/></svg>';
 export const _MODELDIR_CHECK_ON = '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.6" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="9"/><polyline points="8 12 11 15 16 9"/></svg>';
 
+function _normalizeCookbookModelDir(dir) {
+  const d = String(dir || '').replaceAll('✕', '').replaceAll('✖', '').trim();
+  return /^(home|mnt|media|data|opt|srv|var)\//.test(d) ? `/${d}` : d;
+}
+
 // Monochrome platform glyphs (currentColor) for a server's OS tag: a penguin for
 // Linux, the four-pane logo for Windows, an Android robot for Termux/Android.
 function _platformIcon(platform) {
@@ -170,6 +175,92 @@ function _gemma4ThinkingChatTemplateArg(modelName) {
     : '';
 }
 
+const _SERVER_COLOR_CHOICES = [
+  ['', 'Auto'],
+  ['#bd93f9', 'Purple'],
+  ['#ff79c6', 'Pink'],
+  ['#fca5a5', 'Red'],
+  ['#93c5fd', 'Blue'],
+  ['#86efac', 'Green'],
+  ['#d6b37a', 'Bronze'],
+  ['#111827', 'Black'],
+  ['#f8fafc', 'White'],
+  ['#c0c4cc', 'Silver'],
+  ['#d9f99d', 'Lime'],
+  ['#ccfbf1', 'Mint'],
+];
+const _SERVER_AUTO_COLOR_VALUES = _SERVER_COLOR_CHOICES.slice(1).map(([value]) => value);
+
+function _serverColorValue(value) {
+  const v = String(value || '').trim();
+  return /^#[0-9a-fA-F]{6}$/.test(v) ? v : '';
+}
+
+function _serverColor(s) {
+  return _serverColorValue(s && s.color);
+}
+
+function _serverColorLabel(color) {
+  const hit = _SERVER_COLOR_CHOICES.find(([value]) => value === color);
+  return hit ? hit[1] : 'Auto';
+}
+
+function _autoServerColor(index) {
+  const servers = Array.isArray(_envState.servers) ? _envState.servers : [];
+  const explicit = new Set(servers.map(s => _serverColor(s)).filter(Boolean));
+  const used = new Set(explicit);
+  for (let j = 0; j <= index; j++) {
+    const s = servers[j] || {};
+    const explicitColor = _serverColor(s);
+    if (explicitColor) continue;
+    const picked = _SERVER_AUTO_COLOR_VALUES.find(c => !used.has(c)) || _SERVER_AUTO_COLOR_VALUES[j % _SERVER_AUTO_COLOR_VALUES.length] || '';
+    if (j === index) return picked;
+    if (picked) used.add(picked);
+  }
+  return _SERVER_AUTO_COLOR_VALUES[index % _SERVER_AUTO_COLOR_VALUES.length] || '';
+}
+
+function _resolvedServerColor(s, index) {
+  return _serverColor(s) || _autoServerColor(index);
+}
+
+function _serverOptionLabel(label, color) {
+  return color ? `● ${label}` : label;
+}
+
+function _serverOptionStyle(color) {
+  return color ? ` style="color:${esc(color)};"` : '';
+}
+
+function _serverColorOptionStyle(color) {
+  if (!color) return ' style="background:var(--bg);color:var(--fg);"';
+  const c = String(color).toLowerCase();
+  const fg = (c === '#111827') ? '#f8fafc' : (c === '#f8fafc' || c === '#fca5a5' || c === '#93c5fd' || c === '#86efac' || c === '#d9f99d' || c === '#ccfbf1' || c === '#c0c4cc') ? '#111827' : color;
+  return ` style="color:${esc(fg)};background-color:color-mix(in srgb, ${esc(color)} 28%, var(--bg));"`;
+}
+
+function _serverColorForValue(value) {
+  const s = _serverByVal(value);
+  const idx = Array.isArray(_envState.servers) ? _envState.servers.indexOf(s) : -1;
+  return s ? _resolvedServerColor(s, idx >= 0 ? idx : 0) : '';
+}
+
+export function _applyServerSelectColor(sel) {
+  if (!sel || sel.tagName !== 'SELECT') return;
+  const color = _serverColorForValue(sel.value);
+  if (color) {
+    sel.style.setProperty('--cookbook-server-color', color);
+    sel.classList.add('cookbook-server-select-colored');
+  } else {
+    sel.style.removeProperty('--cookbook-server-color');
+    sel.classList.remove('cookbook-server-select-colored');
+  }
+}
+
+export function _syncServerSelectColors(root = document) {
+  root.querySelectorAll('#hwfit-server-select, #hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(_applyServerSelectColor);
+}
+
 function _buildServerOpts(excludeLocal = false) {
   // The local server is ALWAYS represented by the synthetic value="local" option
   // (showing its custom name from the "server name" feature). We must therefore
@@ -177,7 +268,8 @@ function _buildServerOpts(excludeLocal = false) {
   const _localIdx = _envState.servers.findIndex(_isLocalEntry);
   const _localSrv = _localIdx >= 0 ? _envState.servers[_localIdx] : null;
   const _localLabel = (_localSrv && _localSrv.name) ? _localSrv.name : 'Local';
-  let html = `<option value="local"${!_envState.remoteHost ? ' selected' : ''}>${esc(_localLabel)}</option>`;
+  const _localColor = _localSrv ? _resolvedServerColor(_localSrv, _localIdx) : '';
+  let html = `<option value="local"${_serverOptionStyle(_localColor)}${!_envState.remoteHost ? ' selected' : ''}>${esc(_serverOptionLabel(_localLabel, _localColor))}</option>`;
   const selectedKey = _envState.remoteServerKey || '';
   let legacyHostSelected = false;
   for (let i = 0; i < _envState.servers.length; i++) {
@@ -186,12 +278,13 @@ function _buildServerOpts(excludeLocal = false) {
     if (excludeLocal && _isLocalEntry(s)) continue;
     const label = s.name || s.host || `Server ${i + 1}`;
     const value = _serverKey(s);
+    const color = _resolvedServerColor(s, i);
     let selected = selectedKey ? value === selectedKey : false;
     if (!selectedKey && _envState.remoteHost === s.host && !legacyHostSelected) {
       selected = true;
       legacyHostSelected = true;
     }
-    html += `<option value="${esc(value)}"${selected ? ' selected' : ''}>${esc(label)}</option>`;
+    html += `<option value="${esc(value)}"${_serverOptionStyle(color)}${selected ? ' selected' : ''}>${esc(_serverOptionLabel(label, color))}</option>`;
   }
   return html;
 }
@@ -345,6 +438,8 @@ export function _detectReasoningParser(modelName) {
   // MiniMax M2 / M2.5 / M2.7 — released with a dedicated parser. Catch M2
   // before plain "minimax" so M2.x doesn't fall through to a wrong parser.
   if (n.includes('minimax') && n.match(/\bm2(?:\.\d)?\b/)) return 'minimax_m2';
+  // DeepSeek-V4 has a dedicated parser in SGLang. Keep it before R1/V3.
+  if (n.includes('deepseek') && /\bv[-_]?4\b/.test(n)) return 'deepseek-v4';
   // DeepSeek-R1 / V3-Thinking / V3.1-Thinking variants. Bare V3/V3.1 (non-
   // thinking) skip this — they're not reasoning models.
   if (n.includes('deepseek') && (n.includes('r1') || n.includes('thinking'))) return 'deepseek_r1';
@@ -382,6 +477,7 @@ export function _detectToolParser(modelName) {
   if (n.includes('llama-4') || n.includes('llama4')) return 'llama4_json';
   if (n.includes('llama') || n.includes('nemotron')) return 'llama3_json';
   if (n.includes('mistral') || n.includes('mixtral')) return 'mistral';
+  if (n.includes('deepseek') && /\bv[-_]?4\b/.test(n)) return 'deepseekv4';
   if (n.includes('deepseek-v3')) return 'deepseek_v3';
   if (n.includes('deepseek')) return 'deepseek_v3';
   if (n.includes('minimax') && /\bm3\b/.test(n)) return 'minimax_m3';
@@ -409,7 +505,7 @@ export function _detectBackend(model) {
   const isAppleSilicon = ['metal', 'mps', 'apple'].includes(sysBackend);
   const _nm = `${model.repo_id || ''} ${model.path || ''} ${model.name || ''}`.toLowerCase();
   if (/\bmlx\b|mlx-|_mlx/i.test(_nm) || q.startsWith('MLX')) {
-    return { backend: 'unsupported', label: 'Unsupported' };
+    return { backend: 'mlx', label: 'MLX' };
   }
   const isAwqLike = /^AWQ|^GPTQ|^NVFP4/.test(q) || ['FP8', 'FP4', 'MXFP4', 'NF4', 'INT4', 'INT8', 'W4A16', 'W8A8', 'W8A16'].includes(q) || /\b(awq|gptq|fp8|fp4|nvfp4|mxfp4|nf4|int4|int8|w4a16|w8a8|w8a16)\b/i.test(_nm);
   const hasGgufFile = Array.isArray(model.gguf_files)
@@ -442,7 +538,7 @@ export function _detectBackend(model) {
   // don't run on macOS; vLLM-native quantized models are already filtered out
   // of metal Cookbook results, so llama.cpp is always the right engine here.
   if (['metal', 'mps', 'apple'].includes(sysBackend)) {
-    return { backend: 'llamacpp', label: 'llama.cpp' };
+    return { backend: 'mlx', label: 'MLX' };
   }
 
   // ROCm/AMD machines should not blindly default HF safetensors models to
@@ -540,13 +636,32 @@ function _venvRootFromPath(path) {
   return p;
 }
 
+function _venvLooksWrongForPlatform(path, platform) {
+  const p = String(path || '').trim();
+  const plat = String(platform || '').toLowerCase();
+  if (!p || !plat) return false;
+  if ((plat === 'darwin' || plat === 'macos') && /^\/(?:home|usr\/local\/cuda|opt\/conda)\//.test(p)) return true;
+  if ((plat === 'linux' || plat === 'termux') && /^\/(?:Users|opt\/homebrew)\//.test(p)) return true;
+  return false;
+}
+
+function _isDeepSeekV4Model(modelName) {
+  const n = String(modelName || '').toLowerCase();
+  return n.includes('deepseek') && /\bv[-_]?4\b/.test(n);
+}
+
+function _envHasKey(envText, key) {
+  return String(envText || '').split(/\s+/).some(part => part.startsWith(`${key}=`));
+}
+
 export function _buildServeCmd(f, modelName, backend) {
   // When a venv is configured on the chosen server, use the venv's binaries
   // by absolute path. Bare `vllm` / `python3` relies on PATH, and SSH non-
   // interactive sessions often leave a user-site install (~/.local/bin/vllm)
   // ahead of the venv's bin, so the WRONG vllm gets launched even with the
   // venv activated. Absolute path sidesteps the whole PATH question.
-  const _formVenv = (f.venv ?? '').toString().trim();
+  let _formVenv = (f.venv ?? '').toString().trim();
+  if (_venvLooksWrongForPlatform(_formVenv, f.platform)) _formVenv = '';
   const _activeVenvPath = _venvRootFromPath(_formVenv || (_envState.env === 'venv' ? (_envState.envPath || '') : ''));
   const _venvBin = _activeVenvPath ? (_activeVenvPath + '/bin/') : '';
   const _vllmBin = _venvBin ? `${_venvBin}vllm` : 'vllm';
@@ -618,14 +733,19 @@ export function _buildServeCmd(f, modelName, backend) {
     // button strip is the only source for which devices to pin.
     const gpuId = (f.gpus || f.gpu_id || '').toString().trim();
     cmd += _gpuEnvPrefix(gpuId);
-    const _extraEnv = (f.extra_env ?? '').toString().replace(/\s+/g, ' ').trim();
+    const _isDsv4 = _isDeepSeekV4Model(modelName);
+    let _extraEnv = (f.extra_env ?? '').toString().replace(/\s+/g, ' ').trim();
+    if (_isDsv4 && !_envHasKey(_extraEnv, 'SGLANG_DSV4_COMPRESS_STATE_DTYPE')) {
+      _extraEnv = (`SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16 ${_extraEnv}`).trim();
+    }
     if (_extraEnv) cmd += _extraEnv + ' ';
     cmd += `${_py3Bin} -m sglang.launch_server --model-path ${modelName} --host 0.0.0.0 --port ${f.port || '30000'}`;
     const _gemma4ChatTemplate = _gemma4ThinkingChatTemplateArg(modelName);
     if (_gemma4ChatTemplate) cmd += ` --chat-template ${_gemma4ChatTemplate}`;
     if (f.tp && f.tp !== '1') cmd += ` --tp ${f.tp}`;
     if (f.ctx) cmd += ` --context-length ${f.ctx}`;
-    if (f.gpu_mem && f.gpu_mem !== '0.90') cmd += ` --mem-fraction-static ${f.gpu_mem}`;
+    const _memFraction = _isDsv4 && (!f.gpu_mem || f.gpu_mem === '0.90') ? '0.80' : f.gpu_mem;
+    if (_memFraction && _memFraction !== '0.90') cmd += ` --mem-fraction-static ${_memFraction}`;
     if (f.dtype && f.dtype !== 'auto') cmd += ` --dtype ${f.dtype}`;
     if (f.max_seqs && f.max_seqs.toString().trim()) cmd += ` --max-running-requests ${f.max_seqs.toString().trim()}`;
     if (f.trust_remote) cmd += ' --trust-remote-code';
@@ -638,6 +758,14 @@ export function _buildServeCmd(f, modelName, backend) {
     }
     if (!f.prefix_cache) cmd += ' --disable-radix-cache';
     if (f.enforce_eager) cmd += ' --disable-cuda-graph';
+    const _decodeGraph = String(f.sglang_decode_graph || '').trim();
+    if (!f.enforce_eager && _decodeGraph === 'disabled') {
+      cmd += ' --cuda-graph-backend-decode disabled';
+    } else if (!f.enforce_eager && _decodeGraph === 'bs16') {
+      cmd += ' --cuda-graph-max-bs-decode 16';
+    } else if (!f.enforce_eager && _isDsv4 && !/\s--cuda-graph-max-bs-decode\b/.test(cmd) && !/\s--cuda-graph-backend-decode\b/.test(cmd)) {
+      cmd += ' --cuda-graph-backend-decode disabled';
+    }
   } else if (backend === 'llamacpp') {
     const ggufPath = f._gguf_path || 'model.gguf';
     // GPU list — read from gpus (button strip); fall back to gpu_id for
@@ -812,6 +940,13 @@ export function _buildServeCmd(f, modelName, backend) {
     if (f.diff_attention_slicing) cmd += ' --attention-slicing';
     if (f.diff_vae_slicing) cmd += ' --vae-slicing';
     if (f.diff_harmonize_gpu) cmd += ` --harmonize-gpu ${f.diff_harmonize_gpu}`;
+  } else if (backend === 'mlx') {
+    const mlxPy = _isWindows() ? 'python' : _py3Bin;
+    const mlxHost = f.host ? '0.0.0.0' : '127.0.0.1';
+    cmd += `${mlxPy} -m mlx_lm.server --model ${_shellQuote(modelName)} --host ${mlxHost} --port ${f.port || '8080'}`;
+    if (/minimax|mini-max/i.test(modelName)) {
+      cmd += ' --temp 0.7 --top-p 0.9 --max-tokens 2048';
+    }
   }
   return cmd;
 }
@@ -933,6 +1068,7 @@ async function _fetchDependencies() {
     const pkgs = data.packages || [];
     if (!pkgs.length) { list.innerHTML = '<div class="hwfit-loading">No packages found</div>'; return; }
     const _winUnsupported = new Set(['hf_transfer', 'vllm', 'rembg', 'gfpgan']);
+    const _systemInstallable = new Set(['tmux']);
 
     const _statusTag = (pkg, isLocal, isSystemDep, winBlocked) => {
       if (winBlocked) return `<span class="cookbook-dep-tag cookbook-dep-na">N/A</span>`;
@@ -944,8 +1080,12 @@ async function _fetchDependencies() {
       if (pkg.installed) return `<button class="cookbook-dep-tag cookbook-dep-installed cookbook-dep-installed-btn" title="Installed — click for actions"><span class="cookbook-dep-installed-label">Installed</span><span class="cookbook-dep-caret">&#9662;</span></button>`;
       if (isSystemDep) {
         const depTip = esc(pkg.install_hint || 'Install this OS package on the selected server.');
+        if (pkg.applicable !== false && _systemInstallable.has(pkg.name)) {
+          return `<button type="button" class="cookbook-dep-tag cookbook-dep-install cookbook-dep-install-sysdeps" data-dep-sysdeps="${esc(pkg.name)}" data-dep-target="${isLocal ? 'local' : 'remote'}" title="${depTip}">Install</button>`;
+        }
         const depLabel = pkg.applicable === false ? 'N/A ?' : 'Missing';
-        return `<span class="cookbook-dep-tag cookbook-dep-na" title="${depTip}">${depLabel}</span>`;
+        const depStyle = pkg.name === 'docker' ? ' style="width:87.7px;justify-content:center;"' : '';
+        return `<span class="cookbook-dep-tag cookbook-dep-na" title="${depTip}"${depStyle}>${depLabel}</span>`;
       }
       return `<button class="cookbook-dep-tag cookbook-dep-install" data-dep-pip="${esc(pkg.pip)}" data-dep-target="${isLocal ? 'local' : 'remote'}">Install</button>`;
     };
@@ -957,6 +1097,7 @@ async function _fetchDependencies() {
     const _DEP_GLYPHS = {
       vllm: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"/><path d="M14 4l4 9 3-9"/></svg>',
       sglang: '<span aria-hidden="true" style="display:block;width:13px;height:13px;background:currentColor;-webkit-mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;"></span>',
+      mlx_lm: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M4 18V6l4 7 4-7v12"/><path d="M16 6v12"/><path d="M20 6v12"/></svg>',
       llama_cpp: '<svg width="13" height="13" viewBox="0 0 600 600" fill="none" aria-hidden="true"><path d="M600 392L504.249 558L504.137 557.929C487.252 584.069 458.193 600 426.864 600H120L240 392H600Z" fill="currentColor"/><path d="M240 392H0L199.602 46.0254C216.032 17.5463 246.411 0 279.29 0H466.154L240 392Z" fill="currentColor"/></svg>',
       ollama: '<img src="/static/icons/ollama-mark-crop.png" alt="" aria-hidden="true" width="13" height="13" style="display:block;width:13px;height:13px;object-fit:contain;" />',
       diffusers: '<svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"/></svg>',
@@ -1121,22 +1262,32 @@ async function _fetchDependencies() {
     // "Update" item in an installed package's ⋮ menu. `upgrade` adds pip -U;
     // `statusEl`, when given, shows "Installing…/Updating…" and is disabled.
     async function _installDep(pipName, pkgName, isLocalOnly, upgrade, statusEl) {
+      let targetServer = null;
       if (isLocalOnly) {
         _envState.remoteHost = '';
         _envState.env = 'none';
         _envState.envPath = '';
       } else {
         const depsServerSel = document.getElementById('hwfit-deps-server');
-        if (depsServerSel) _applyServerSelection(depsServerSel.value);
+        if (depsServerSel) {
+          targetServer = _serverByVal(depsServerSel.value);
+          _applyServerSelection(depsServerSel.value);
+        }
       }
-      const targetHost = isLocalOnly ? 'this server' : (_envState.remoteHost || 'local');
+      const targetHost = isLocalOnly ? 'this server' : ((targetServer?.host || _envState.remoteHost) || 'local');
+      const targetEnv = isLocalOnly ? 'none' : (targetServer?.env || _envState.env || 'none');
+      const targetEnvPath = isLocalOnly ? '' : (targetServer?.envPath || _envState.envPath || '');
+      const targetPlatform = isLocalOnly ? (_envState.hostPlatform || _envState.platform || '') : (targetServer?.platform || _envState.platform || '');
+      const targetRemoteHost = isLocalOnly ? '' : (targetServer?.host || _envState.remoteHost || '');
       // Always go through `python -m pip` so the leading token is `python`
       // — matches the /api/model/serve allow-list (bare `pip` is blocked).
       // Inside a venv/conda env, `--user` is invalid (pip refuses), so we
       // only add `--user --break-system-packages` when there's no env —
       // for PEP-668-locked system pythons (Arch, newer Debian).
-      const _inEnv = _envState.env === 'venv' || _envState.env === 'conda';
+      const _inEnv = targetEnv === 'venv' || targetEnv === 'conda';
-      const _pipFlags = (!_isWindows() && !_inEnv) ? ' --user --break-system-packages' : '';
+      const _platform = String(targetPlatform || '').toLowerCase();
+      const _isAppleTarget = _platform === 'darwin' || _platform === 'macos' || _platform.includes('mac os');
+      const _pipFlags = (!_isWindows() && !_inEnv) ? (_isAppleTarget ? ' --user' : ' --user --break-system-packages') : '';
       // Use the venv's python3 by absolute path when configured. Even with the
       // env_prefix sourcing activate, SSH non-interactive sessions sometimes
       // pick a `python3` ahead of the venv's bin on PATH, so the install
@@ -1144,35 +1295,35 @@ async function _fetchDependencies() {
       let _py;
       if (_isWindows()) {
         _py = 'python';
-      } else if (_envState.env === 'venv' && _envState.envPath) {
+      } else if (targetEnv === 'venv' && targetEnvPath) {
-        _py = `${_envState.envPath.replace(/\/+$/, '')}/bin/python3`;
+        _py = `${targetEnvPath.replace(/\/+$/, '')}/bin/python3`;
       } else {
         _py = 'python3';
       }
       const cmd = `${_py} -m pip install${upgrade ? ' -U' : ''}${_pipFlags} "${pipName}"`;
       let envPrefix = '';
       if (_isWindows()) {
-        if (_envState.env === 'venv' && _envState.envPath) {
+        if (targetEnv === 'venv' && targetEnvPath) {
-          envPrefix = '& ' + _psQuote(_envState.envPath.endsWith('\\Scripts\\Activate.ps1') ? _envState.envPath : _envState.envPath + '\\Scripts\\Activate.ps1');
+          envPrefix = '& ' + _psQuote(targetEnvPath.endsWith('\\Scripts\\Activate.ps1') ? targetEnvPath : targetEnvPath + '\\Scripts\\Activate.ps1');
-        } else if (_envState.env === 'conda' && _envState.envPath) {
+        } else if (targetEnv === 'conda' && targetEnvPath) {
-          envPrefix = 'conda activate ' + _psQuote(_envState.envPath);
+          envPrefix = 'conda activate ' + _psQuote(targetEnvPath);
         }
       } else {
-        if (_envState.env === 'venv' && _envState.envPath) {
+        if (targetEnv === 'venv' && targetEnvPath) {
-          const p = _envState.envPath;
+          const p = targetEnvPath;
           envPrefix = 'source ' + _shellQuote(p.endsWith('/bin/activate') ? p : p + '/bin/activate');
-        } else if (_envState.env === 'conda' && _envState.envPath) {
+        } else if (targetEnv === 'conda' && targetEnvPath) {
-          envPrefix = 'eval "$(conda shell.bash hook)" && conda activate ' + _shellQuote(_envState.envPath);
+          envPrefix = 'eval "$(conda shell.bash hook)" && conda activate ' + _shellQuote(targetEnvPath);
         }
       }
       try {
         const reqBody = {
           repo_id: pipName,
           cmd: cmd,
-          remote_host: _envState.remoteHost || undefined,
+          remote_host: targetRemoteHost || undefined,
-          ssh_port: _getPort(_envState.remoteHost) || undefined,
+          ssh_port: _getPort(targetRemoteHost) || undefined,
           env_prefix: envPrefix || undefined,
-          platform: _envState.platform || undefined,
+          platform: targetPlatform || undefined,
         };
         const res = await fetch('/api/model/serve', {
           method: 'POST', credentials: 'same-origin',
@@ -1196,7 +1347,7 @@ async function _fetchDependencies() {
         }
         // _dep flags this as a pip dependency/driver install (not a servable
         // model) so the running-task card doesn't offer a "Serve →" button.
-        const payload = { repo_id: pipName, _cmd: cmd, remote_host: _envState.remoteHost || '', _dep: true, env_path: _envState.envPath || '' };
+        const payload = { repo_id: pipName, _cmd: cmd, remote_host: targetRemoteHost || '', _dep: true, env_path: targetEnvPath || '', platform: targetPlatform || '' };
         _addTask(data.session_id, 'pip ' + pkgName, 'download', payload);
         if (statusEl) { statusEl.textContent = upgrade ? 'Updating...' : 'Installing...'; statusEl.disabled = true; }
         uiModule.showToast(`${upgrade ? 'Updating' : 'Installing'} ${pkgName} on ${targetHost}...`);
@@ -1334,7 +1485,7 @@ async function _fetchDependencies() {
             // from the row) so the user can copy-paste it without leaving
             // the toast. Otherwise just surface the error.
             const _suffix = _resolvedCmd ? `\n\nRun on ${targetLabel}: ${_resolvedCmd}` : '';
-            uiModule.showToast('Build-deps install failed: ' + String(reason).slice(0, 300) + _suffix, {
+            uiModule.showToast('System dependency install failed: ' + String(reason).slice(0, 300) + _suffix, {
               duration: 25000,
               action: _resolvedCmd ? 'Copy command' : 'OK',
               onAction: async () => {
@@ -1638,9 +1789,47 @@ function _applyServerSelection(val) {
       sel.value = _want;
       if (sel.selectedIndex < 0) sel.value = 'local';
     }
+    _applyServerSelectColor(sel);
   });
 }
 
+async function _refreshScanDownloadTarget() {
+  const btn = document.getElementById('hwfit-hw-refresh-btn');
+  if (btn && btn.disabled) return;
+  const selectedVal = document.getElementById('hwfit-server-select')?.value || _currentServerValue();
+  if (btn) {
+    btn.disabled = true;
+    btn.style.opacity = '0.55';
+    btn.style.cursor = 'wait';
+  }
+  try {
+    if (selectedVal) _applyServerSelection(selectedVal);
+    const ok = await _syncFromServer().catch((e) => {
+      console.warn('[cookbook] explicit server sync failed', e);
+      return false;
+    });
+    if (ok) {
+      try { Object.assign(_envState, _readStoredEnvState()); } catch {}
+      if (selectedVal) _applyServerSelection(selectedVal);
+    }
+    _resetGpuToggleState();
+    await Promise.allSettled([
+      _hwfitFetch(true),
+      _fetchCachedModels(true),
+    ]);
+    if (uiModule?.showToast) uiModule.showToast('Refreshed selected server');
+  } catch (e) {
+    console.warn('[cookbook] scan/download refresh failed', e);
+    if (uiModule?.showError) uiModule.showError('Refresh failed: ' + (e?.message || e));
+  } finally {
+    if (btn) {
+      btn.disabled = false;
+      btn.style.opacity = '';
+      btn.style.cursor = '';
+    }
+  }
+}
+
 function _wireTabEvents(body) {
   // Tab switching
   body.querySelectorAll('.cookbook-tab').forEach(tab => {
@@ -1701,17 +1890,18 @@ function _wireTabEvents(body) {
       const port = entry.querySelector('.cookbook-srv-port')?.value?.trim() || '';
       const env = entry.querySelector('.cookbook-srv-env')?.value || 'none';
       const envPath = entry.querySelector('.cookbook-srv-path')?.value?.trim() || '';
+      const color = _serverColorValue(entry.querySelector('.cookbook-srv-color')?.value || '');
       const platform = entry.dataset.platform || '';
       const dirs = [];
       entry.querySelectorAll('.cookbook-modeldir-tag').forEach(tag => {
         // Read from data attribute (authoritative) — never parse displayed text
-        const d = (tag.dataset.dir || '').replaceAll('✕', '').replaceAll('✖', '').trim();
+        const d = _normalizeCookbookModelDir(tag.dataset.dir || '');
         if (d) dirs.push(d);
       });
       // Directory flagged as the download target ('' = default HF cache).
       const dlEl = entry.querySelector('.cookbook-modeldir-dl.active');
       const downloadDir = dlEl ? (dlEl.dataset.dlDir || '') : '';
-      servers.push({ name, host, port, env, envPath, modelDirs: dirs, downloadDir, platform });
+      servers.push({ name, host, port, env, envPath, color, modelDirs: dirs, downloadDir, platform });
     });
     _envState.servers = servers;
     // Auto-default: when the user has configured EXACTLY ONE remote server
@@ -1737,13 +1927,16 @@ function _wireTabEvents(body) {
     Promise.resolve().then(() => {
       const _want = _currentServerValue();
       document.querySelectorAll('#hwfit-server-select, #hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(sel => {
-        if (sel && sel.tagName === 'SELECT') sel.value = _want;
+        if (sel && sel.tagName === 'SELECT') {
+          sel.value = _want;
+          _applyServerSelectColor(sel);
+        }
       });
     });
   }
 
   // Wire server form inputs
-  document.querySelectorAll('.cookbook-srv-name, .cookbook-srv-host, .cookbook-srv-port, .cookbook-srv-path').forEach(el => {
+  document.querySelectorAll('.cookbook-srv-name, .cookbook-srv-color, .cookbook-srv-host, .cookbook-srv-port, .cookbook-srv-path').forEach(el => {
     el.addEventListener('change', _syncServers);
   });
   document.querySelectorAll('.cookbook-srv-env').forEach(el => {
@@ -1788,7 +1981,7 @@ function _wireTabEvents(body) {
       if (cacheDirEl) cacheDirEl.value = srv.modelDir || '~/.cache/huggingface/hub';
       const dirsEl = document.querySelector('.cookbook-serve-dirs');
       if (dirsEl) {
-        const dirs = (Array.isArray(srv.modelDirs) ? srv.modelDirs : [srv.modelDir || '~/.cache/huggingface/hub']).map(d => d.replaceAll('✕', '').replaceAll('✖', '').trim()).filter(Boolean);
+        const dirs = (Array.isArray(srv.modelDirs) ? srv.modelDirs : [srv.modelDir || '~/.cache/huggingface/hub']).map(d => _normalizeCookbookModelDir(d)).filter(Boolean);
         dirsEl.innerHTML = dirs.map(d => `<span class="cookbook-serve-dir-pill">${esc(d)}</span>`).join('') +
           '<span class="cookbook-serve-dir-edit" title="Edit in Settings">edit</span>';
         dirsEl.querySelector('.cookbook-serve-dir-edit')?.addEventListener('click', () => {
@@ -1802,7 +1995,22 @@ function _wireTabEvents(body) {
 
   const scanBtn = document.getElementById('hwfit-cache-scan');
   if (scanBtn) {
-    scanBtn.addEventListener('click', () => _fetchCachedModels(true));
+    scanBtn.addEventListener('click', async () => {
+      if (scanBtn.disabled) return;
+      scanBtn.disabled = true;
+      scanBtn.classList.add('spinning');
+      try {
+        await _fetchCachedModels(true);
+      } finally {
+        scanBtn.disabled = false;
+        scanBtn.classList.remove('spinning');
+      }
+    });
+  }
+
+  const hwRefreshBtn = document.getElementById('hwfit-hw-refresh-btn');
+  if (hwRefreshBtn) {
+    hwRefreshBtn.addEventListener('click', _refreshScanDownloadTarget);
   }
 
   const editDirsLink = document.querySelector('.cookbook-serve-dir-edit');
@@ -1822,6 +2030,7 @@ function _wireTabEvents(body) {
       _fetchDependencies();
     });
   }
+  _syncServerSelectColors(body);
 
   // "Rebuild llama.cpp" clears the cached build so the next serve recompiles.
   // The serve bootstrap only builds llama-server when it is missing from PATH,
@@ -2522,11 +2731,30 @@ function _wireTabEvents(body) {
 // (Model Directory header, default-server checkmark, trash delete, platform icon).
 // forceRemote renders an editable remote entry even before a host is typed
 // (a new server's host is empty, which would otherwise read as "Local").
+export function _serverDefaultHtml(active) {
+  const check = active ? '<span class="hwfit-hf-check cookbook-srv-default-check" title="Default server" style="font-weight:800;color:var(--green,#50fa7b);font-size:15px;line-height:1;flex-shrink:0;position:relative;top:2px;">✓</span>' : '';
+  return `${check}<span class="cookbook-srv-default-label">default</span>`;
+}
+
 export function _serverEntryHtml(s, i, defaultServer, forceRemote, isNew) {
   const isLocal = (forceRemote || isNew) ? false : (!s.host || s.host === 'local');
   const envOpts = [['none', 'None'], ['venv', 'venv'], ['conda', 'conda']].map(([value, label]) => `<option value="${value}"${s.env === value ? ' selected' : ''}>${label}</option>`).join('');
+  const srvColor = _serverColor(s);
+  const resolvedSrvColor = _resolvedServerColor(s, i);
+  const colorOpts = _SERVER_COLOR_CHOICES.map(([value, label]) => {
+    const displayLabel = label;
+    return `<option value="${esc(value)}"${_serverColorOptionStyle(value)}${srvColor === value ? ' selected' : ''}>${esc(displayLabel)}</option>`;
+  }).join('');
+  const selectedColorLabel = srvColor ? _serverColorLabel(srvColor) : `Auto · ${_serverColorLabel(resolvedSrvColor)}`;
+  const colorMenu = _SERVER_COLOR_CHOICES.map(([value, label]) => {
+    const active = value === srvColor;
+    const swatchColor = value || resolvedSrvColor;
+    const rowLabel = value ? label : `Auto · ${_serverColorLabel(resolvedSrvColor)}`;
+    const swatch = swatchColor ? ` style="--swatch-color:${esc(swatchColor)};"` : '';
+    return `<button type="button" class="cookbook-srv-color-item${active ? ' active' : ''}" data-color="${esc(value)}"${swatch}><span class="cookbook-srv-color-item-dot"></span><span>${esc(rowLabel)}</span></button>`;
+  }).join('');
   let html = '';
-  html += `<div class="cookbook-server-entry" data-idx="${i}" data-platform="${esc(s.platform || '')}">`;
+  html += `<div class="cookbook-server-entry" data-idx="${i}" data-platform="${esc(s.platform || '')}"${resolvedSrvColor ? ` style="--cookbook-server-color:${esc(resolvedSrvColor)};"` : ''}>`;
   const _srvTitle = s.name || (isLocal ? 'Local' : (s.host || `Server ${i + 1}`));
   const _srvKey = isLocal ? 'local' : (s.host || '');
   const _isDefaultSrv = (defaultServer || '') === _srvKey;
@@ -2542,12 +2770,13 @@ export function _serverEntryHtml(s, i, defaultServer, forceRemote, isNew) {
     // sense once the server is saved.
     html += `<span style="margin-left:auto;display:inline-flex;gap:4px;align-items:center;">${_checkBtn}${_keyBtn}<button class="cookbook-server-cancel-btn" title="Discard this new server" style="height:22px;box-sizing:border-box;display:inline-flex;align-items:center;position:relative;top:-2px;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>Cancel</button></span>`;
   } else {
-    html += `<span style="margin-left:auto;display:inline-flex;gap:4px;align-items:center;">${!isLocal ? _checkBtn + _keyBtn : ''}<span class="cookbook-srv-default${_isDefaultSrv ? ' active' : ''}" title="${_isDefaultSrv ? 'Default server — Cookbook opens here' : 'Make this the default server'}" data-srv-key="${esc(_srvKey)}">${_isDefaultSrv ? _MODELDIR_CHECK_ON : _MODELDIR_CHECK_OFF}<span class="cookbook-srv-default-label">default</span></span></span>`;
+    html += `<span style="margin-left:auto;display:inline-flex;gap:4px;align-items:center;">${!isLocal ? _checkBtn + _keyBtn : ''}<span class="cookbook-srv-default${_isDefaultSrv ? ' active' : ''}" title="${_isDefaultSrv ? 'Default server — Cookbook opens here' : 'Make this the default server'}" data-srv-key="${esc(_srvKey)}">${_serverDefaultHtml(_isDefaultSrv)}</span></span>`;
   }
   html += `</span>`;
   html += `<div class="cookbook-server-row">`;
   html += `<input type="text" class="hwfit-sf cookbook-srv-name" value="${esc(s.name || (isLocal ? 'Local' : ''))}" placeholder="Name (optional)" style="width:92px;flex-shrink:0;" />`;
-  html += `<input type="text" class="hwfit-sf cookbook-srv-host" value="${isLocal ? '' : esc(s.host || '')}" placeholder="e.g. user@ip" style="width:214.5px;flex-shrink:0;box-sizing:border-box;" ${isLocal ? 'readonly' : ''} />`;
+  html += `<span class="cookbook-srv-color-wrap has-color" title="Server color"><select class="hwfit-sf cookbook-srv-color" aria-hidden="true" tabindex="-1">${colorOpts}</select><button type="button" class="hwfit-sf cookbook-srv-color-btn" aria-haspopup="listbox" aria-expanded="false"><span class="cookbook-srv-color-dot" aria-hidden="true"></span><span class="cookbook-srv-color-label">${esc(selectedColorLabel)}</span><svg class="cookbook-srv-color-caret" width="9" height="9" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.6" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="6 9 12 15 18 9"/></svg></button><div class="cookbook-srv-color-menu hidden" role="listbox">${colorMenu}</div></span>`;
+  html += `<input type="text" class="hwfit-sf cookbook-srv-host" value="${isLocal ? '' : esc(s.host || '')}" placeholder="e.g. user@ip" style="width:184px;flex-shrink:0;box-sizing:border-box;" ${isLocal ? 'readonly' : ''} />`;
   html += `<input type="text" class="hwfit-sf cookbook-srv-port" value="${esc(s.port || '')}" placeholder="Port" title="SSH port (default 22)" style="width:48px;flex-shrink:0;" ${isLocal ? 'readonly' : ''} />`;
   html += `<select class="hwfit-sf cookbook-srv-env">${envOpts}</select>`;
   html += `<input type="text" class="hwfit-sf cookbook-srv-path" value="${esc(s.envPath || '')}" placeholder="${s.platform === 'windows' ? 'venv/conda env' : '~/venv or conda-env'}" />`;
@@ -2567,13 +2796,15 @@ export function _serverEntryHtml(s, i, defaultServer, forceRemote, isNew) {
     html += `<span class="cookbook-modeldir-tag${isDefault ? ' cookbook-modeldir-default' : ''}${isTarget ? ' cookbook-modeldir-target' : ''}" data-dir-idx="${j}" data-dir="${esc(modelDirs[j])}">${dlBtn} ${esc(modelDirs[j])}${rmBtn}</span>`;
   }
   html += `<button class="cookbook-modeldir-add" title="Add model directory">+ Add</button>`;
-  const _btnStyle = 'margin-left:auto;position:relative;top:-2px;height:22px;box-sizing:border-box;display:inline-flex;align-items:center;';
+  const _btnBaseStyle = 'position:relative;top:-2px;height:22px;box-sizing:border-box;display:inline-flex;align-items:center;';
+  const _btnPushStyle = `margin-left:auto;${_btnBaseStyle}`;
   if (isNew) {
     // A brand-new server: Save (confirm) sits where Delete would be; Cancel is
     // top-right in the title. Save confirms with a checkmark (auto-saves on edit too).
-    html += `<button class="cookbook-server-save-btn" title="Save this server" style="${_btnStyle}"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><polyline points="17 21 17 13 7 13 7 21"/><polyline points="7 3 7 8 15 8"/></svg>Save</button>`;
+    html += `<button class="cookbook-server-save-btn" title="Save this server" style="${_btnPushStyle}"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><polyline points="17 21 17 13 7 13 7 21"/><polyline points="7 3 7 8 15 8"/></svg>Save</button>`;
   } else if (!isLocal) {
-    html += `<button class="cookbook-server-rm cookbook-server-rm-btn" title="Delete this server" style="${_btnStyle}"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><path d="M3 6h18"/><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6"/></svg>Delete</button>`;
+    html += `<button class="cookbook-server-rm cookbook-server-rm-btn" title="Delete this server" style="${_btnPushStyle}"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><path d="M3 6h18"/><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"/><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6"/></svg>Delete</button>`;
+    html += `<button class="cookbook-server-save-btn" title="Save server changes" style="${_btnBaseStyle}"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:4px;flex-shrink:0;"><path d="M19 21H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11l5 5v11a2 2 0 0 1-2 2z"/><polyline points="17 21 17 13 7 13 7 21"/><polyline points="7 3 7 8 15 8"/></svg>Save</button>`;
   }
   html += `</div>`;
   if (!isLocal) {
@@ -2709,6 +2940,7 @@ function _renderRecipes() {
   html += '<option value="">Engine</option>';
   html += '<option value="llamacpp">llama.cpp</option>';
   html += '<option value="ollama">Ollama</option>';
+  html += '<option value="mlx">MLX</option>';
   html += '<option value="vllm">vLLM</option>';
   html += '<option value="sglang">SGLang</option>';
   html += '<option value="diffusers">Diffusers</option>';
@@ -2726,7 +2958,7 @@ function _renderRecipes() {
   html += '<span class="hwfit-quant-wrap">';
   html += '<select class="cookbook-field-input hwfit-quant" id="hwfit-quant" style="height:28px;">';
   html += '<option value="" selected>Quant</option>';
-  html += '<option value="Q4_K_M">Q4</option><option value="Q8_0">Q8</option>';
+  html += '<option value="Q4_K_M">Q4 / AWQ</option><option value="Q8_0">Q8</option>';
   html += '<option value="Q6_K">Q6</option><option value="Q5_K_M">Q5</option>';
   html += '<option value="Q3_K_M">Q3</option><option value="Q2_K">Q2</option>';
   html += '<option value="AWQ-4bit">AWQ</option><option value="FP8">FP8</option><option value="FP4">FP4</option><option value="NVFP4">NVFP4</option></select>';
@@ -2744,9 +2976,8 @@ function _renderRecipes() {
   html += _buildServerOpts(false);
   html += '</select>';
   html += '<div class="hwfit-gpu-toggles" id="hwfit-gpu-toggles"></div>';
-  // (Rescan button removed — Edit handles manual hardware updates;
-  // automatic re-probe runs on container restart.)
   html += '<button type="button" class="hwfit-gpu-btn hwfit-hw-manual-btn" id="hwfit-hw-manual-btn" title="Set hardware manually" style="flex-shrink:0;position:relative;top:-3px;left:-1px;display:inline-flex;align-items:center;gap:3px;"><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" style="flex-shrink:0;"><path d="M12 20h9"/><path d="M16.5 3.5a2.121 2.121 0 0 1 3 3L7 19l-4 1 1-4Z"/></svg>EDIT</button>';
+  html += '<button type="button" class="hwfit-gpu-btn hwfit-hw-refresh-btn" id="hwfit-hw-refresh-btn" title="Refresh selected server hardware and cached models" aria-label="Refresh selected server hardware and cached models" style="flex-shrink:0;position:relative;top:-3px;left:-3px;width:26px;height:26px;padding:0;display:inline-flex;align-items:center;justify-content:center;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M1 4v6h6"/><path d="M23 20v-6h-6"/><path d="M20.49 9A9 9 0 0 0 5.64 5.64L1 10"/><path d="M3.51 15a9 9 0 0 0 14.85 3.36L23 14"/></svg></button>';
   // Sort state — the clickable column headers read/write this (pewds' original
   // sort paradigm). Newest is reachable by clicking the Model column header.
   html += '<select class="cookbook-field-input hwfit-sort" id="hwfit-sort" style="display:none">';
@@ -2787,7 +3018,7 @@ function _renderRecipes() {
   html += '<h2 style="margin:0;padding:0;line-height:1;">Serve <span id="serve-stats" class="memory-count" style="font-size:0.6em;opacity:0.6;font-weight:normal"></span></h2>';
   html += '</div>';
   const _selSrv = _es.servers.find(s => s.host === _es.remoteHost) || _es.servers[0] || {};
-  const _srvDirs = (Array.isArray(_selSrv.modelDirs) ? _selSrv.modelDirs : [_selSrv.modelDir || '~/.cache/huggingface/hub']).map(d => d.replaceAll('✕', '').replaceAll('✖', '').trim()).filter(Boolean);
+  const _srvDirs = (Array.isArray(_selSrv.modelDirs) ? _selSrv.modelDirs : [_selSrv.modelDir || '~/.cache/huggingface/hub']).map(d => _normalizeCookbookModelDir(d)).filter(Boolean);
   html += '<div class="cookbook-serve-dirs" style="margin-top:6px;">';
   html += _srvDirs.map(d => `<span class="cookbook-serve-dir-pill">${esc(d)}</span>`).join('');
   html += '<span class="cookbook-serve-dir-edit" title="Edit in Settings">edit</span>';
@@ -2797,6 +3028,7 @@ function _renderRecipes() {
   html += '<select class="memory-sort-select" id="serve-sort" style="height:24px;">';
   html += '<option value="name">Name</option><option value="size-desc">Size \u2193</option><option value="size-asc">Size \u2191</option><option value="recent">Recent</option>';
   html += '</select>';
+  html += '<button type="button" class="hwfit-gpu-btn" id="hwfit-cache-scan" title="Refresh cached models on selected server" aria-label="Refresh cached models on selected server"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M1 4v6h6"/><path d="M23 20v-6h-6"/><path d="M20.49 9A9 9 0 0 0 5.64 5.64L1 10"/><path d="M3.51 15a9 9 0 0 0 14.85 3.36L23 14"/></svg></button>';
   html += '</div>';
   html += '<div class="memory-toolbar" style="margin-top:8px;">';
   html += '<div class="memory-category-filters">';
@@ -3170,6 +3402,7 @@ document.addEventListener('cookbook:state-synced', () => {
   if (isVisible()) {
     const activeTab = document.querySelector('#cookbook-modal .cookbook-tab.active')?.dataset?.backend || '';
     if (activeTab === 'Running') _renderRunningTab();
+    else if (activeTab === 'Serve') _rerenderCachedModels();
   }
 });
 

static/js/cookbookDownload.js

@@ -484,8 +484,12 @@ export async function _runModelDownload(panel, model, backend, hostOverride) {
   // they disagree on the active host. The servers LIST is consistent, so we look
   // up the matching server to get its env / path / platform / port.
   let host;
+  let selectedServer = null;
+  let selectedServerKey = '';
   if (hostOverride !== undefined) {
     host = hostOverride || '';
+    selectedServer = host ? (_serverByVal?.(host) || (_envState.servers || []).find(s => s.host === host) || null) : null;
+    selectedServerKey = selectedServer ? (typeof window.cookbookModule?._serverKey === 'function' ? window.cookbookModule._serverKey(selectedServer) : '') : '';
   } else {
     // No explicit host passed: resolve from the visible server dropdown rather
     // than _envState.remoteHost (unreliable — multiple state copies disagree).
@@ -496,15 +500,21 @@ export async function _runModelDownload(panel, model, backend, hostOverride) {
     const _dsrv = (_ssv && _ssv !== 'local') ? (_serverByVal?.(_ssv) || _envState.servers[parseInt(_ssv)]) : null;
     if (_dsrv) {
       host = _dsrv.host;
+      selectedServer = _dsrv;
+      selectedServerKey = _ssv || '';
     } else if (ssEl && ssEl.value === 'local') {
       host = '';
     } else {
       host = _envState.remoteHost || '';
+      selectedServer = host ? ((_envState.servers || []).find(s => s.host === host) || _serverByVal?.(host) || null) : null;
     }
   }
-  const srv = _serverByVal?.(_envState.remoteServerKey || host) || {};
+  const srv = selectedServer || _serverByVal?.(host) || {};
-  const env = host ? (srv.env || 'none') : (_envState.env || 'none');
+  let env = host ? (srv.env || 'none') : (_envState.env || 'none');
   const envPath = host ? (srv.envPath || '') : (_envState.envPath || '');
+  if ((!env || env === 'none') && envPath) {
+    env = /(?:^|\/)(?:\.?venv|env)(?:\/|$)|\/bin\/activate$/i.test(envPath) ? 'venv' : env;
+  }
   const platform = host ? (srv.platform || '') : (_envState.platform || '');
   const isWin = host ? (platform === 'windows') : _isWindows();
 
@@ -515,7 +525,13 @@ export async function _runModelDownload(panel, model, backend, hostOverride) {
   // resumes cached partials more reliably.
   if ((model.required_gb || 0) >= 10 || backend === 'llamacpp') payload.disable_hf_transfer = true;
   if (_envState.hfToken) payload.hf_token = _envState.hfToken;
-  if (host) { payload.remote_host = host; const _sp = _getPort(host); if (_sp) payload.ssh_port = _sp; }
+  if (host) {
+    payload.remote_host = host;
+    if (selectedServerKey && selectedServerKey !== 'local') payload.remote_server_key = selectedServerKey;
+    if (srv.name) payload.remote_server_name = srv.name;
+    const _sp = srv.port || _getPort(host);
+    if (_sp) payload.ssh_port = _sp;
+  }
   if (platform) payload.platform = platform;
   // If this server has a directory flagged as the download target, send it so
   // the backend downloads into <dir>/<model> instead of the default HF cache.
@@ -562,11 +578,12 @@ export async function _runModelDownload(panel, model, backend, hostOverride) {
   if (zombieCandidate) {
     try {
       const _zh = zombieCandidate.remoteHost || '';
-      const _zPort = (_serverByVal?.(_envState.remoteServerKey || _zh)
+      const _zPort = (_serverByVal?.(zombieCandidate.remoteServerKey || zombieCandidate.payload?.remote_server_key || _zh)
         || (_envState.servers || []).find(s => s.host === _zh) || {}).port;
       const _sshPf = _zh ? `ssh ${_zPort && _zPort !== '22' ? `-p ${_zPort} ` : ''}${_zh} '` : '';
       const _sshSf = _zh ? `'` : '';
-      const _probeCmd = `${_sshPf}tmux has-session -t ${zombieCandidate.sessionId} 2>/dev/null${_sshSf}`;
+      const _probePrefix = _zh ? 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; ' : '';
+      const _probeCmd = `${_sshPf}${_probePrefix}tmux has-session -t ${zombieCandidate.sessionId} 2>/dev/null${_sshSf}`;
       const _r = await fetch('/api/shell/exec', {
         method: 'POST', credentials: 'same-origin',
         headers: { 'Content-Type': 'application/json' },
@@ -593,7 +610,7 @@ export async function _runModelDownload(panel, model, backend, hostOverride) {
   if (activeOnHost) {
     const queueId = `queue-${Date.now().toString(36)}`;
     const allTasks = _loadTasks();
-    allTasks.push({ id: queueId, sessionId: queueId, name: taskName, type: 'download', status: 'queued', output: '', ts: Date.now(), payload, remoteHost: host });
+    allTasks.push({ id: queueId, sessionId: queueId, name: taskName, type: 'download', status: 'queued', output: '', ts: Date.now(), payload, remoteHost: host, remoteServerKey: payload.remote_server_key || '', remoteServerName: payload.remote_server_name || '', sshPort: payload.ssh_port || '', platform: payload.platform || '' });
     _saveTasks(allTasks);
     _renderRunningTab();
     uiModule.showToast(`Queued ${shortName} — waiting for current download`);

static/js/cookbookRunning.js

@@ -57,9 +57,24 @@ function _downloadDisplayName(name, task) {
   return part ? `${name} · ${part}` : name;
 }
 
+function _downloadNameFromPayload(name, payload) {
+  const rawName = String(name || '').trim();
+  // Defensive: failed/restarted downloads can inherit the wrapper executable
+  // name if older state was saved from a command preview. The row title should
+  // always be the model/repo, never "bash" or "python".
+  const looksLikeLauncher = /^(?:bash|sh|zsh|python|python3|pwsh|powershell|cmd|tmux)$/i.test(rawName);
+  const base = (!rawName || looksLikeLauncher)
+    ? String(payload?.repo_id || payload?.repo || '').split('/').pop()
+    : rawName;
+  const include = payload?.include || '';
+  if (!include || String(base || '').includes(' · ')) return base || rawName || 'download';
+  const part = _ggufDisplayPartFromPath(String(include).replace(/\*/g, ''));
+  return part ? `${base} · ${part}` : (base || rawName || 'download');
+}
+
 function _taskDisplayName(task) {
   const name = String(task?.name || '').trim();
-  if (task?.type === 'download') return _downloadDisplayName(name, task);
+  if (task?.type === 'download') return _downloadDisplayName(_downloadNameFromPayload(name, task?.payload), task);
   if (task?.type !== 'serve') return name;
   const gguf = task?.payload?._fields?.gguf_file || task?.payload?.gguf_file || '';
   if (!gguf || name.includes(' · ')) return name;
@@ -98,7 +113,7 @@ function _downloadOutputLooksActive(task) {
 
 function _canClearTask(task) {
   if (!task || task.status === 'running') return false;
-  if (task.type === 'serve' && (task.status === 'ready' || (task._serveReady && !['stopped', 'error', 'crashed', 'failed', 'completed'].includes(task.status)))) return false;
+  if (task.type === 'serve' && (task.status === 'ready' || (!['error', 'crashed', 'failed', 'completed'].includes(task.status) && _serveOutputLooksReady(task)))) return false;
   // If the tmux output still shows an in-flight download, the task isn't
   // actually finished — hide the clear/check pill so it doesn't show on a
   // task that's still doing work. (The next render will reflect this and
@@ -334,6 +349,34 @@ function _taskServerSelection(task) {
   return { host, server, key };
 }
 
+function _serverColorForTaskGroup(key, tasks) {
+  const firstTask = Array.isArray(tasks) ? tasks[0] : null;
+  const host = firstTask?.remoteHost || firstTask?.payload?.remote_host || '';
+  const savedKey = firstTask?.remoteServerKey || firstTask?.payload?.remote_server_key || key || '';
+  const server = (savedKey ? _serverByVal?.(savedKey) : null)
+    || (key ? _serverByVal?.(key) : null)
+    || (host ? _serverByVal?.(host) : null)
+    || (key === 'local' || !key ? (_envState?.servers || []).find(s => !s.host || String(s.host).toLowerCase() === 'local') : null)
+    || null;
+  const color = String(server?.color || '').trim();
+  return /^#[0-9a-fA-F]{6}$/.test(color) ? color : '';
+}
+
+function _serverHeaderStyle(color) {
+  if (!color) return '';
+  const c = color.toLowerCase();
+  const accent = (c === '#ffffff' || c === '#f8fafc') ? '#cbd5e1'
+    : (c === '#111827' || c === '#000000') ? '#64748b'
+    : color;
+  return ` style="--cookbook-server-color:${esc(color)};--cookbook-server-accent:${esc(accent)};"`;
+}
+
+function _shouldAutoExpandTaskOutput(task) {
+  return task?.type === 'download'
+    && !task?.payload?._dep
+    && ['running', 'queued', 'error', 'crashed'].includes(String(task?.status || ''));
+}
+
 function _selectTaskServer(task) {
   const { host, server, key } = _taskServerSelection(task);
   _envState.remoteHost = host;
@@ -366,6 +409,7 @@ let _soloExpandTaskId = null;
 const TASKS_KEY = 'cookbook-tasks';
 const STORAGE_KEY = 'cookbook-presets';
 const SERVE_STATE_KEY = 'cookbook-serve-state';
+const SERVE_FAVORITES_KEY = 'cookbook-serve-favorite-models';
 
 // Polling / timeout intervals
 const TASK_POLL_INTERVAL_MS = 3000;       // delay between reconnect-loop iterations
@@ -489,7 +533,7 @@ function _refreshModelsAfterEndpointChange() {
     pickerLabel.innerHTML = '<span style="opacity:0.4;">refreshing…</span>';
   }
   if (window.modelsModule && window.modelsModule.refreshModels) {
-    window.modelsModule.refreshModels(true);
+    window.modelsModule.refreshModels(false);
   }
   setTimeout(() => {
     if (!window.sessionModule) return;
@@ -545,6 +589,53 @@ function _endpointFromAdvertisedUrl(rawUrl, currentHost, fallbackPort = '11434')
   }
 }
 
+function _serveExpectedModel(task) {
+  const fields = task?.payload?._fields || {};
+  return String(
+    fields.served_model_name ||
+    fields.model_path ||
+    task?.payload?.repo_id ||
+    task?.model ||
+    task?.name ||
+    ''
+  ).trim();
+}
+
+function _modelIdMatchesExpected(modelId, expected) {
+  const got = String(modelId || '').trim().toLowerCase();
+  const want = String(expected || '').trim().toLowerCase();
+  if (!got || !want) return true;
+  if (got === want) return true;
+  const gotBase = got.split('/').pop();
+  const wantBase = want.split('/').pop();
+  return gotBase === wantBase || got.includes(wantBase) || want.includes(gotBase);
+}
+
+function _endpointMatchesServe(ep, task) {
+  const expected = _serveExpectedModel(task);
+  const models = [...(ep?.models || []), ...(ep?.pinned_models || [])];
+  if (!models.length) return true;
+  return models.some(mid => _modelIdMatchesExpected(mid, expected));
+}
+
+function _markServeEndpointMismatch(task, ep, host, port) {
+  const expected = _serveExpectedModel(task);
+  const actual = (ep?.models || []).join(', ') || 'no models';
+  const msg = `Port ${host}:${port} answered, but it is serving ${actual}, not ${expected || task?.name || 'the launched model'}. The new serve likely failed or the port is occupied by an older server.`;
+  _updateTask(task.sessionId || task.session_id, {
+    status: 'error',
+    _serveReady: false,
+    _endpointAdded: false,
+    output: `${task.output || ''}\n\n${msg}`.trim(),
+  });
+  uiModule.showError(msg);
+}
+
+function _appendPinnedServeModel(fd, task) {
+  const expected = _serveExpectedModel(task);
+  if (expected) fd.append('pinned_models', expected);
+}
+
 // ── Download queue — runs one at a time per server ──
 
 function _processQueue() {
@@ -891,13 +982,17 @@ function _taskRemoteHost(task) {
   return task?.remoteHost || task?.payload?.remote_host || '';
 }
 
+function _remoteTmuxPrefix() {
+  return 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; ';
+}
+
 export function _tmuxCmd(task, tmuxArgs) {
   if (_isWindows(task)) {
     return _winSessionCmd(task, tmuxArgs);
   }
   const host = _taskRemoteHost(task);
   if (host) {
-    return `ssh ${_sshPrefix(_getPort(task))}${host} 'tmux ${tmuxArgs}' 2>/dev/null`;
+    return `ssh ${_sshPrefix(_getPort(task))}${host} '${_remoteTmuxPrefix()}tmux ${tmuxArgs}' 2>/dev/null`;
   }
   return `tmux ${tmuxArgs} 2>/dev/null`;
 }
@@ -930,7 +1025,7 @@ function _winSessionCmd(task, tmuxArgs) {
       : `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`;
     return _winPowerShellCmd(task, ps);
   }
-  return host ? `ssh ${pf}${host} 'tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`;
+  return host ? `ssh ${pf}${host} '${_remoteTmuxPrefix()}tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`;
 }
 
 function _winPowerShellCmd(task, ps) {
@@ -957,7 +1052,7 @@ export function _tmuxGracefulKill(task) {
   }
   const host = _taskRemoteHost(task);
   if (host) {
-    return `ssh ${_sshPrefix(_getPort(task))}${host} 'tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
+    return `ssh ${_sshPrefix(_getPort(task))}${host} '${_remoteTmuxPrefix()}tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
   }
   return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`;
 }
@@ -984,7 +1079,7 @@ export function _tmuxForceKill(task) {
     `tmux kill-session -t ${sid} 2>/dev/null`;
   const host = _taskRemoteHost(task);
   if (host) {
-    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(inner)}`;
+    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(_remoteTmuxPrefix() + inner)}`;
   }
   return inner;
 }
@@ -1001,7 +1096,7 @@ export function _tmuxIsAliveCheck(task) {
   const inner = `if tmux has-session -t ${sid} 2>/dev/null; then echo ALIVE; else echo DEAD; fi`;
   const host = _taskRemoteHost(task);
   if (host) {
-    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(inner)}`;
+    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(_remoteTmuxPrefix() + inner)}`;
   }
   return inner;
 }
@@ -1225,8 +1320,13 @@ function _syncToServer() {
         presets: _loadPresets(),
         env: _envState,
         serveState: null,
+        serveFavorites: [],
       };
       try { state.serveState = JSON.parse(localStorage.getItem(SERVE_STATE_KEY)); } catch {}
+      try {
+        const favorites = JSON.parse(localStorage.getItem(SERVE_FAVORITES_KEY) || '[]');
+        state.serveFavorites = Array.isArray(favorites) ? favorites.filter(Boolean).map(String) : [];
+      } catch {}
       await fetch('/api/cookbook/state', {
         method: 'POST', credentials: 'same-origin',
         headers: { 'Content-Type': 'application/json' },
@@ -1236,6 +1336,10 @@ function _syncToServer() {
   }, 400);
 }
 
+document.addEventListener('cookbook:state-dirty', () => {
+  _syncToServer();
+});
+
 // Normalize state from server: collapse legacy duplicate keys to canonical form.
 // - server.modelDir (singular) → server.modelDirs[0] (canonical)
 // - strip ✕/✖ pollution from modelDirs
@@ -1317,6 +1421,9 @@ export async function _syncFromServer() {
     if (state.serveState) {
       localStorage.setItem(SERVE_STATE_KEY, JSON.stringify(state.serveState));
     }
+    if (Array.isArray(state.serveFavorites)) {
+      localStorage.setItem(SERVE_FAVORITES_KEY, JSON.stringify(state.serveFavorites.filter(Boolean).map(String)));
+    }
     document.dispatchEvent(new CustomEvent('cookbook:state-synced', { detail: state }));
     return true;
   } catch { return false; }
@@ -1384,6 +1491,7 @@ async function _retryDownload(name, payload, replaceSessionId = '') {
       const tasks = _loadTasks();
       const task = tasks.find(t => t.sessionId === replaceSessionId);
       if (task) {
+        task.name = _downloadNameFromPayload(name || task.name, _payload);
         task.id = data.session_id;
         task.sessionId = data.session_id;
         task.status = 'running';
@@ -1510,6 +1618,11 @@ export async function _serveAutoRetryReplace(panel, flag, value) {
   _animateOutThenRemove(taskEl, taskId);
 
   let newCmd = task.payload._cmd;
+  if (flag === '--cuda-graph-backend-decode') {
+    newCmd = newCmd.replace(/\s+--cuda-graph-max-bs-decode(?:\s+\S+|=\S+)/g, '');
+  } else if (flag === '--cuda-graph-max-bs-decode') {
+    newCmd = newCmd.replace(/\s+--cuda-graph-backend-decode(?:\s+\S+|=\S+)/g, '');
+  }
   const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
   if (re.test(newCmd)) {
     newCmd = newCmd.replace(re, `${flag} ${value}`);
@@ -1641,6 +1754,7 @@ function _parseServeCmdToFields(cmd) {
   const ex = (re) => { const m = cmd.match(re); return m ? m[1] : ''; };
   const fields = {
     backend: cmd.includes('llama_cpp') || cmd.includes('llama-server') ? 'llamacpp'
+      : cmd.includes('mlx_lm.server') ? 'mlx'
       : cmd.includes('diffusion_server') ? 'diffusers'
       : cmd.includes('sglang') ? 'sglang'
       : cmd.includes('ollama') ? 'ollama' : 'vllm',
@@ -1678,6 +1792,100 @@ function _parseServeCmdToFields(cmd) {
   return fields;
 }
 
+function _serveCmdNeedsGpuPreflight(cmd, repo) {
+  const c = String(cmd || '').toLowerCase();
+  const r = String(repo || '').toLowerCase();
+  if (!c || /gpu-cleanup|sglang-kernel|mlx-lm|pip\s+install|python\d*\s+-m\s+pip/.test(`${r} ${c}`)) return false;
+  return /\b(vllm\s+serve|sglang(?:\.launch_server|\s+serve)|mlx_lm\.server|llama-server|llama_cpp\.server|text-generation-launcher|aphrodite|ollama\s+(?:serve|run))\b/.test(c);
+}
+
+function _selectedGpuIndexes(gpus) {
+  const raw = String(gpus || '').trim();
+  if (!raw) return null;
+  const out = new Set();
+  raw.split(',').forEach(part => {
+    const p = part.trim();
+    const range = p.match(/^(\d+)\s*-\s*(\d+)$/);
+    if (range) {
+      const a = parseInt(range[1], 10);
+      const b = parseInt(range[2], 10);
+      for (let i = Math.min(a, b); i <= Math.max(a, b); i++) out.add(i);
+      return;
+    }
+    const n = parseInt(p, 10);
+    if (Number.isFinite(n)) out.add(n);
+  });
+  return out.size ? out : null;
+}
+
+function _gbFromMb(mb) {
+  const n = Number(mb || 0);
+  if (!Number.isFinite(n) || n <= 0) return '';
+  return n >= 1024 ? `${(n / 1024).toFixed(n >= 10240 ? 0 : 1)}G` : `${Math.round(n)}M`;
+}
+
+function _gpuPreflightIssues(data, selected) {
+  const backend = String(data?.backend || data?.source || '').toLowerCase();
+  const isCuda = backend.includes('cuda') || String(data?.source || '').toLowerCase().includes('nvidia');
+  const rows = Array.isArray(data?.gpus) ? data.gpus : [];
+  const issues = [];
+  rows.forEach(g => {
+    const idx = Number(g?.index);
+    if (selected && !selected.has(idx)) return;
+    const procs = Array.isArray(g?.processes) ? g.processes : [];
+    if (procs.length) {
+      procs.slice(0, 3).forEach(p => {
+        const name = String(p?.name || 'process').split(/[\\/]/).pop();
+        const used = _gbFromMb(p?.used_mb);
+        issues.push(`GPU ${idx}: ${name}${p?.pid ? ` #${p.pid}` : ''}${used ? ` (${used})` : ''}`);
+      });
+      if (procs.length > 3) issues.push(`GPU ${idx}: +${procs.length - 3} more process${procs.length - 3 === 1 ? '' : 'es'}`);
+      return;
+    }
+    const total = Number(g?.total_mb || 0);
+    const free = Number(g?.free_mb || 0);
+    const used = Number(g?.used_mb || 0);
+    const freeRatio = total > 0 ? free / total : 1;
+    // CUDA can have display/runtime crumbs; warn only for meaningful occupied memory.
+    if (isCuda && used > 4096 && freeRatio < 0.9) {
+      issues.push(`GPU ${idx}: ${_gbFromMb(used)} already used (${_gbFromMb(free)} free)`);
+    } else if (!isCuda && total > 0 && freeRatio < 0.2) {
+      issues.push(`${g?.name || `GPU ${idx}`}: low free memory (${_gbFromMb(free)} free of ${_gbFromMb(total)})`);
+    } else if (!isCuda && g?.busy && total <= 0) {
+      issues.push(`${g?.name || `GPU ${idx}`}: GPU device is busy`);
+    }
+  });
+  return issues;
+}
+
+async function _confirmGpuPreflight(reqBody, shortName, repo, cmd) {
+  if (!_serveCmdNeedsGpuPreflight(cmd, repo)) return true;
+  const params = new URLSearchParams();
+  if (reqBody.remote_host) params.set('host', reqBody.remote_host);
+  if (reqBody.ssh_port) params.set('ssh_port', reqBody.ssh_port);
+  try {
+    const res = await fetch(`/api/cookbook/gpus${params.toString() ? `?${params.toString()}` : ''}`, {
+      method: 'GET',
+      credentials: 'same-origin',
+    });
+    const data = await res.json().catch(() => null);
+    if (!res.ok || !data?.ok) return true;
+    const selected = _selectedGpuIndexes(reqBody.gpus);
+    const issues = _gpuPreflightIssues(data, selected);
+    if (!issues.length) return true;
+    const where = reqBody.remote_host || 'local';
+    const list = issues.slice(0, 6).join('; ');
+    const more = issues.length > 6 ? `; +${issues.length - 6} more` : '';
+    const msg = `GPU preflight found existing load on ${where}: ${list}${more}. Launch ${shortName || 'model'} anyway?`;
+    const confirm = window.styledConfirm || uiModule?.styledConfirm;
+    if (confirm) return await confirm(msg, { confirmText: 'Launch anyway', cancelText: 'Cancel' });
+    return window.confirm ? window.confirm(msg) : true;
+  } catch (e) {
+    console.warn('[cookbook] GPU preflight failed; allowing launch', e);
+    return true;
+  }
+}
+
 export async function _launchServeTask(shortName, repo, cmd, fields, hostOverride, targetMeta = null) {
   // Host resolution mirrors the download path: when the caller passes an explicit
   // host (resolved from the dropdown the user actually picked), use it and look
@@ -1761,11 +1969,16 @@ export async function _launchServeTask(shortName, repo, cmd, fields, hostOverrid
     ssh_port: _getPort(_serverMetaKey || _host) || undefined,
     env_prefix: envPrefix || undefined,
     hf_token: _envState.hfToken || undefined,
-    gpus: _envState.gpus || undefined,
+    gpus: _usedGpus || undefined,
     platform: _hplatform || undefined,
   };
 
   try {
+    const _preflightOk = await _confirmGpuPreflight(reqBody, shortName, repo, cmd);
+    if (!_preflightOk) {
+      uiModule.showToast('Launch cancelled — GPU is already in use');
+      return;
+    }
     const res = await fetch('/api/model/serve', {
       method: 'POST', credentials: 'same-origin',
       headers: { 'Content-Type': 'application/json' },
@@ -1989,7 +2202,8 @@ export function _renderRunningTab() {
       // green when reachable, red if any serve task on it is crashed/unreachable.
       const _secDot = (key && allTasks.some(_serveTaskFailed)) ? 'fail' : 'ok';
       const _dotTitle = key ? (_secDot === 'fail' ? 'Server not responding' : 'Reachable') : 'Local (this machine)';
-      sec.insertAdjacentHTML('afterbegin', `<div class="cookbook-section-header" data-collapse="${bodyId}"><svg class="cookbook-section-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><polyline points="6 9 12 15 18 9"/></svg><span class="cookbook-srv-status ${_secDot}" title="${_dotTitle}" style="flex-shrink:0;position:relative;top:0px;"></span><span class="cookbook-section-title" style="margin:0;">${esc(sg.name)}</span><button class="cookbook-btn cookbook-stop-all-btn" data-stop-server="${esc(key)}" title="Stop all running servers"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" stroke="none" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><rect x="5" y="5" width="14" height="14" rx="1.5"/></svg>Stop all</button><button class="cookbook-btn cookbook-clear-btn" data-clear-server="${esc(key)}" title="Clear finished tasks"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/></svg>Clear finished</button></div><div id="${bodyId}" class="cookbook-section-body"></div>`);
+      const _srvColor = _serverColorForTaskGroup(key || 'local', allTasks);
+      sec.insertAdjacentHTML('afterbegin', `<div class="cookbook-section-header${_srvColor ? ' has-server-color' : ''}" data-collapse="${bodyId}"${_serverHeaderStyle(_srvColor)}><svg class="cookbook-section-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round"><polyline points="6 9 12 15 18 9"/></svg><span class="cookbook-srv-status ${_secDot}" title="${_dotTitle}" style="flex-shrink:0;position:relative;top:0px;"></span><span class="cookbook-section-title" style="margin:0;">${esc(sg.name)}</span><button class="cookbook-btn cookbook-stop-all-btn" data-stop-server="${esc(key)}" title="Stop all running servers"><svg width="11" height="11" viewBox="0 0 24 24" fill="currentColor" stroke="none" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><rect x="5" y="5" width="14" height="14" rx="1.5"/></svg>Stop all</button><button class="cookbook-btn cookbook-clear-btn" data-clear-server="${esc(key)}" title="Clear finished tasks"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="vertical-align:-1px;margin-right:4px;"><polyline points="3 6 5 6 21 6"/><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/><path d="M10 11v6"/><path d="M14 11v6"/></svg>Clear finished</button></div><div id="${bodyId}" class="cookbook-section-body"></div>`);
     }
   }
 
@@ -2170,7 +2384,7 @@ export function _renderRunningTab() {
         <button class="cookbook-task-menu-btn" title="Actions">&#8942;</button>
       </div>
       <div class="cookbook-task-sub"><span class="cookbook-task-session">${esc(task.sessionId)}</span><span class="cookbook-task-uptime" style="display:${((task.type === 'serve' || task.type === 'download') && task.status === 'running') ? '' : 'none'}"></span>${(task.type === 'download') ? `<span class="cookbook-task-dldir" title="Download destination" style="font-size:9px;color:var(--fg-muted);font-family:'Fira Code',monospace;opacity:0.4;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;max-width:40ch;">Dir: ${esc(task.payload?.local_dir || '~/.cache/huggingface/hub')}</span>` : ''}</div>
-      <div class="cookbook-output-wrap cookbook-task-collapsible${_mobileCollapseDefault ? ' cookbook-task-collapsed' : ''}"><pre class="cookbook-output-pre">${esc(task.output || '')}</pre><button type="button" class="copy-code cookbook-output-copy"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button></div>
+      <div class="cookbook-output-wrap cookbook-task-collapsible${(_mobileCollapseDefault && !_shouldAutoExpandTaskOutput(task)) ? ' cookbook-task-collapsed' : ''}"><pre class="cookbook-output-pre">${esc(task.output || '')}</pre><button type="button" class="copy-code cookbook-output-copy"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button></div>
     `;
 
     const _waveEl = el.querySelector('.cookbook-task-wave');
@@ -2305,7 +2519,23 @@ export function _renderRunningTab() {
     el.querySelector('.cookbook-task-header').addEventListener('click', (e) => {
       if (e.target.closest('button')) return;
       const wrap = el.querySelector('.cookbook-output-wrap');
-      if (wrap) wrap.classList.toggle('cookbook-task-collapsed');
+      if (!wrap) return;
+      const isOpening = wrap.classList.contains('cookbook-task-collapsed');
+      wrap.classList.toggle('cookbook-task-collapsed');
+      if (isOpening) {
+        _expandedTaskIds.add(task.sessionId);
+        _collapsedTaskIds.delete(task.sessionId);
+        if (task.sessionId && ['serve', 'download'].includes(task.type || '')) {
+          _reconnectTask(el, task);
+        }
+      } else {
+        _collapsedTaskIds.add(task.sessionId);
+        _expandedTaskIds.delete(task.sessionId);
+        if (el._abort) {
+          try { el._abort.abort(); } catch {}
+          el._abort = null;
+        }
+      }
     });
 
     // Wire menu button (also fire from a long-press anywhere on the card so
@@ -2344,10 +2574,18 @@ export function _renderRunningTab() {
       el.addEventListener('touchcancel', _lpCancel, { passive: true });
       menuBtn.addEventListener('click', (e) => {
         e.stopPropagation();
+        const existing = document.querySelector('.cookbook-task-dropdown');
+        if (existing && existing._anchor === menuBtn) {
+          if (typeof existing._dismiss === 'function') existing._dismiss();
+          else existing.remove();
+          return;
+        }
         document.querySelectorAll('.cookbook-task-dropdown').forEach(d => { if (typeof d._dismiss === 'function') d._dismiss(); else d.remove(); });
 
         const dropdown = document.createElement('div');
         dropdown.className = 'cookbook-task-dropdown';
+        dropdown._anchor = menuBtn;
+        menuBtn.classList.add('cookbook-menu-active');
 
         const items = [];
         // ── Run section ─────────────────────────────────────────────
@@ -2549,7 +2787,7 @@ export function _renderRunningTab() {
         }
 
         const closeHandler = (ev) => {
-          if (!dropdown.contains(ev.target) && ev.target !== menuBtn) {
+          if (!dropdown.contains(ev.target) && ev.target !== menuBtn && !menuBtn.contains(ev.target)) {
             _cleanup();
           }
         };
@@ -2561,6 +2799,7 @@ export function _renderRunningTab() {
         const _cleanup = () => {
           _unreg(); _unreg = () => {};
           dropdown.remove();
+          menuBtn.classList.remove('cookbook-menu-active');
           document.removeEventListener('click', closeHandler);
           window.removeEventListener('scroll', scrollClose, true);
           window.visualViewport?.removeEventListener('scroll', scrollClose);
@@ -2732,7 +2971,9 @@ export function _renderRunningTab() {
     // responds; without this, the user opens the Running tab and sees
     // only the placeholder ("Launched by scheduled task …") because
     // _reconnectTask never fires for status 'ready'/'loading'/'warming'.
-    if (_isRunningTabVisible() && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status)) {
+    const _wrapForStream = el.querySelector('.cookbook-output-wrap');
+    const _streamExpanded = _wrapForStream && !_wrapForStream.classList.contains('cookbook-task-collapsed');
+    if (_isRunningTabVisible() && _streamExpanded && task.sessionId && ['serve', 'download'].includes(task.type || '')) {
       _reconnectTask(el, task);
     }
   }
@@ -2764,13 +3005,19 @@ export function _renderRunningTab() {
 // ── Reconnect task (polling loop) ──
 
 async function _reconnectTask(el, task) {
+  if (!el || !task) return;
+  const wrap = el.querySelector('.cookbook-output-wrap');
+  if (!_isRunningTabVisible() || !wrap || wrap.classList.contains('cookbook-task-collapsed')) return;
+  if (el._abort && !el._abort.signal?.aborted) return;
   const output = el.querySelector('.cookbook-output-pre');
+  if (!output) return;
   const controller = new AbortController();
   el._abort = controller;
   let failCount = 0;
 
   while (!controller.signal.aborted) {
-    if (!el.isConnected) {
+    const liveWrap = el.querySelector('.cookbook-output-wrap');
+    if (!el.isConnected || !_isRunningTabVisible() || !liveWrap || liveWrap.classList.contains('cookbook-task-collapsed')) {
       controller.abort();
       break;
     }
@@ -3358,13 +3605,17 @@ async function _reconnectTask(el, task) {
                 // endpoints server-side. Mark so we don't retry, but STILL
                 // refresh the picker (and probe until online) so the new model
                 // shows up without the user having to manually refresh.
+                const _ex = eps.find(e => e.base_url === baseUrl);
+                if (_ex && !_endpointMatchesServe(_ex, task)) {
+                  _markServeEndpointMismatch(task, _ex, host, port);
+                  return null;
+                }
                 task._endpointAdded = true;
                 _updateTask(task.sessionId, { _endpointAdded: true });
                 _autoSaveWorkingConfig(task);   // endpoint live → remember these settings
-                if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
+                if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
                 if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
                 window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', { detail: { baseUrl, host, port, model: task.name } }));
-                const _ex = eps.find(e => e.base_url === baseUrl);
                 if (_ex && _ex.id && !(_ex.models || []).length) _probeEndpointUntilOnline(_ex.id, host, port);
                 return null;
               }
@@ -3374,6 +3625,7 @@ async function _reconnectTask(el, task) {
               fd.append('name', task.name);
               fd.append('skip_probe', 'true');
               _appendCookbookEndpointScope(fd, task.remoteHost || '');
+              _appendPinnedServeModel(fd, task);
               if (_isDiffusion) fd.append('model_type', 'image');
               return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
             })
@@ -3392,7 +3644,7 @@ async function _reconnectTask(el, task) {
                 }
                 window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', { detail: { baseUrl, host, port, model: task.name } }));
                 const _trySelectModel = async (attempt) => {
-                  if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
+                  if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
                   const items = window.modelsModule?.getCachedItems?.() || [];
                   for (const item of items) {
                     if (item.offline) continue;
@@ -3479,6 +3731,16 @@ function _isRunningTabVisible() {
   return activeTab === 'Running';
 }
 
+function _isCookbookVisible() {
+  try {
+    if (window.cookbookModule && typeof window.cookbookModule.isVisible === 'function') {
+      return !!window.cookbookModule.isVisible();
+    }
+  } catch (_) {}
+  const modal = document.getElementById('cookbook-modal');
+  return !!modal && !modal.classList.contains('hidden');
+}
+
 function _foregroundChatBusy() {
   try {
     return !!window.__odysseusChatBusy || Date.now() < (window.__odysseusChatBusyUntil || 0);
@@ -3780,6 +4042,7 @@ function _stopBackgroundMonitor() {
 // the endpoint reports models, then refreshes the picker. Bounded so a
 // genuinely-dead server doesn't poll forever.
 async function _probeEndpointUntilOnline(epId, host, port) {
+  if (!_isCookbookVisible() || _foregroundChatBusy()) return;
   if (!epId) return;
   // Big models (e.g. 70B+) can take several minutes to load weights before
   // the server answers /v1/models. Probe for up to ~5 min, easing the
@@ -3788,6 +4051,7 @@ async function _probeEndpointUntilOnline(epId, host, port) {
   for (let i = 0; i < MAX_TRIES; i++) {
     const interval = i < 12 ? 5000 : 10000;   // 5s for the first minute, then 10s
     await new Promise(r => setTimeout(r, interval));
+    if (!_isCookbookVisible() || _foregroundChatBusy()) return;
     try {
       // Hit the probe endpoint — it re-probes server-side and updates
       // cached_models. We consume (and discard) the SSE stream.
@@ -3797,7 +4061,7 @@ async function _probeEndpointUntilOnline(epId, host, port) {
       const eps = await fetch('/api/model-endpoints', { credentials: 'same-origin' }).then(r => r.json()).catch(() => []);
       const ep = (eps || []).find(e => e.id === epId);
       if (ep && (ep.models || []).length) {
-        if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
+        if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
         if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
         window.dispatchEvent(new CustomEvent('ge:model-endpoints-updated', {
           detail: { baseUrl: ep.base_url || `http://${host}:${port}/v1`, host, port, model: (ep.models || [])[0] || '' },
@@ -3891,7 +4155,8 @@ async function _pollBackgroundStatus() {
         // "stopped" by the backend (its pip package is never in the HF cache the
         // dead-session check inspects). Recover "done" from the retained output's
         // exit-0 sentinel so a clean install isn't downgraded to crashed.
-        const depDone = !!task.payload?._dep && _depInstallSucceeded(task.output);
+        const combinedOutput = `${task.output || ''}\n${live.output_tail || ''}`;
+        const depDone = !!task.payload?._dep && _depInstallSucceeded(combinedOutput);
         // A finished model download whose tmux pane is gone is also reported
         // "stopped" (the dead-session check can miss the landed snapshot).
         // Recover "done" from the terminal `DOWNLOAD_OK` sentinel — emitted
@@ -3901,19 +4166,29 @@ async function _pollBackgroundStatus() {
         // off the conclusive exit sentinel only, never the `/snapshots/` path,
         // which can be printed mid-stream for multi-file downloads.
         const downloadDone = task.type === 'download'
-          && String(task.output || '').includes('DOWNLOAD_OK');
+          && String(combinedOutput || '').includes('DOWNLOAD_OK');
-        const nextStatus = live.status === 'completed'
+        const serveReady = task.type === 'serve'
+          && (live.status === 'ready' || _serveOutputLooksReady({ ...task, output: live.output_tail || task.output || '' }));
+        const completedByOutput = depDone || downloadDone;
+        const nextStatus = completedByOutput
+          ? 'done'
+          : (serveReady
+          ? 'ready'
+          : (live.status === 'completed'
           ? 'done'
           : (live.status === 'error'
             ? 'error'
             : (live.status === 'stopped'
                 ? ((depDone || downloadDone) ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped'))
-                : null));
+                : null))));
         if (nextStatus && task.status !== nextStatus) {
           updates.status = nextStatus;
           if (nextStatus === 'done' && task.payload?._dep) completedDeps.push(task);
         }
-        if ((live.status === 'running' || live.status === 'ready') && task.status !== live.status) {
+        if (serveReady && !task._serveReady) {
+          updates._serveReady = true;
+        }
+        if ((live.status === 'running' || live.status === 'ready') && task.status !== live.status && !serveReady && !completedByOutput) {
           updates.status = live.status === 'ready' ? 'ready' : 'running';
         }
         if (live.progress && live.progress !== task.progress) updates.progress = live.progress;
@@ -3993,6 +4268,11 @@ async function _pollBackgroundStatus() {
           const hostPort = `${host}:${port}`;
           const existing = eps.find(e => e.base_url === baseUrl || e.base_url.includes(hostPort) || e.name === t.model);
           if (existing) {
+            const taskForMatch = localTask || { sessionId: t.session_id, name: t.model, model: t.model, payload: { repo_id: t.model, _cmd } };
+            if (!_endpointMatchesServe(existing, taskForMatch)) {
+              _markServeEndpointMismatch(taskForMatch, existing, host, port);
+              return null;
+            }
             // Already registered — but it may be showing offline because
             // it was added while the server was still warming. Kick a
             // re-probe so it flips online without manual toggle.
@@ -4004,6 +4284,7 @@ async function _pollBackgroundStatus() {
           fd.append('name', t.model);
           fd.append('skip_probe', 'true');
           _appendCookbookEndpointScope(fd, localTask?.remoteHost || t.remote || '');
+          _appendPinnedServeModel(fd, localTask || { name: t.model, model: t.model, payload: { repo_id: t.model, _cmd } });
           if (_isDiffusion) fd.append('model_type', 'image');
           if (_supportsTools) fd.append('supports_tools', 'true');
           return fetch('/api/model-endpoints', { method: 'POST', credentials: 'same-origin', body: fd });
@@ -4016,7 +4297,7 @@ async function _pollBackgroundStatus() {
             // probe, so it lands "offline". Retry-probe in the background
             // until /v1/models responds — no manual enable/disable needed.
             if (data && data.id) _probeEndpointUntilOnline(data.id, host, port);
-            if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(true);
+            if (window.modelsModule?.refreshModels) await window.modelsModule.refreshModels(false);
             if (window.sessionModule?.updateModelPicker) window.sessionModule.updateModelPicker();
           }
         })

static/js/cookbookServe.js

@@ -49,11 +49,25 @@ let _cachedAllModels = [];
 const _CACHED_MODELS_SCAN_KEY = 'cookbook_cached_models_scan_v1';
 const _CACHED_MODELS_SCAN_TTL = 6 * 3600 * 1000;
 
+function _normalizeCookbookModelDir(dir) {
+  const d = String(dir || '').replaceAll('✕', '').replaceAll('✖', '').trim();
+  return /^(home|mnt|media|data|opt|srv|var)\//.test(d) ? `/${d}` : d;
+}
+
 function _readCachedModelScan(sig) {
   try {
     const all = JSON.parse(localStorage.getItem(_CACHED_MODELS_SCAN_KEY) || '{}');
     const entry = all[sig];
-    if (entry && Date.now() - (entry.ts || 0) < _CACHED_MODELS_SCAN_TTL) return entry.data || null;
+    if (entry && Date.now() - (entry.ts || 0) < _CACHED_MODELS_SCAN_TTL) {
+      const data = entry.data || null;
+      const models = Array.isArray(data?.models) ? data.models : [];
+      const staleDownloading = models.some(m =>
+        (m?.status === 'downloading' || m?.has_incomplete) && !_isActivelyDownloading(m?.repo_id)
+      );
+      if (!staleDownloading) return data;
+      delete all[sig];
+      localStorage.setItem(_CACHED_MODELS_SCAN_KEY, JSON.stringify(all));
+    }
   } catch {}
   return null;
 }
@@ -83,6 +97,7 @@ function _loadServeFavorites() {
 function _saveServeFavorites(favorites) {
   try {
     localStorage.setItem(SERVE_FAVORITES_KEY, JSON.stringify(Array.from(favorites || [])));
+    document.dispatchEvent(new CustomEvent('cookbook:state-dirty', { detail: { key: SERVE_FAVORITES_KEY } }));
   } catch {}
 }
 
@@ -218,7 +233,21 @@ function _shellSplitForPreview(cmd) {
 }
 
 function _formatServeCmdPreview(cmd) {
-  const raw = String(cmd || '');
+  let raw = String(cmd || '');
+  const mlxDeepSeekV4Compat = /\bmlx_lm\.server\b/i.test(raw)
+    && /--model\s+['"]?mlx-community\/[^'"\s]*deepseek-v4/i.test(raw);
+  if (mlxDeepSeekV4Compat) {
+    const modelMatch = raw.match(/--model\s+(['"]?)(mlx-community\/[^'"\s]*deepseek-v4[^'"\s]*)\1/i);
+    const homeMatch = raw.match(/((?:\/Users|\/home)\/[^/\s'"]+)/);
+    const shortName = modelMatch?.[2]?.split('/').pop();
+    if (homeMatch && shortName) {
+      const shimPath = `${homeMatch[1]}/.cache/odysseus/mlx-shims/${shortName}`;
+      raw = raw.replace(
+        /--model\s+(['"]?)mlx-community\/[^'"\s]*deepseek-v4[^'"\s]*\1/i,
+        `--model '${shimPath}'`
+      );
+    }
+  }
   if (raw.startsWith('MODEL_FILE=$({')) {
     const marker = /&&\s+([A-Za-z_][A-Za-z0-9_]*=\S+\s+)*(?:[A-Za-z_][A-Za-z0-9_]*=\S+\s+)?(?:llama-server|python3?\s+-m\s+llama_cpp\.server)\b/;
     const match = raw.match(marker);
@@ -233,7 +262,7 @@ function _formatServeCmdPreview(cmd) {
   const lines = [];
   let i = 0;
   while (i < tokens.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(tokens[i])) {
-    lines.push(tokens[i]);
+    lines.push(`export ${tokens[i]}`);
     i++;
   }
   if (tokens[i]) {
@@ -254,11 +283,43 @@ function _formatServeCmdPreview(cmd) {
       lines.push(t);
     }
   }
-  return lines.join('\n');
+  const envCount = lines.findIndex(line => !line.startsWith('export '));
+  const firstCmdLine = envCount < 0 ? lines.length : envCount;
+  const formatted = lines.map((line, idx) => {
+    const isCommandPart = idx >= firstCmdLine;
+    const hasNextCommandPart = lines.slice(idx + 1).some(next => !next.startsWith('export '));
+    return isCommandPart && hasNextCommandPart ? `${line} \\` : line;
+  }).join('\n');
+  if (mlxDeepSeekV4Compat) {
+    return [
+      '# Odysseus runtime compatibility: using sanitized MLX DeepSeek-V4 shim.',
+      formatted,
+    ].join('\n');
+  }
+  return formatted;
 }
 
 function _normalizeServeCmdForLaunch(cmd) {
-  return String(cmd || '')
+  let raw = String(cmd || '');
+  const lines = raw.split(/\r?\n/)
+    .map(s => s.trim().replace(/\s*\\$/, '').trim())
+    .filter(s => s && !s.startsWith('#'));
+  if (lines.some(line => /^(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*=/.test(line))) {
+    const env = [];
+    const body = [];
+    for (const line of lines) {
+      const m = line.match(/^export\s+([A-Za-z_][A-Za-z0-9_]*=.*)$/);
+      if (m) {
+        env.push(m[1]);
+      } else if (/^[A-Za-z_][A-Za-z0-9_]*=\S+$/.test(line)) {
+        env.push(line);
+      } else {
+        body.push(line);
+      }
+    }
+    raw = [...env, ...body].join(' ');
+  }
+  return raw
     .replace(/MODEL_FILE=\$\(\{\s+/g, 'MODEL_FILE=$({ ')
     .replace(/\s+\}\s+\|\s+head\s+-1\)/g, ' } | head -1)')
     .replace(/\s*;\s*/g, '; ')
@@ -567,7 +628,13 @@ function _selectedServeTarget(panel) {
       host = server?.host || '';
     }
   }
-  const venv = panel?.querySelector('[data-field="venv"]')?.value?.trim() || server?.envPath || _envState.envPath || '';
+  const typedVenv = panel?.querySelector('[data-field="venv"]')?.value?.trim() || '';
+  // For remote targets the server profile is authoritative. Otherwise a stale
+  // venv typed/loaded for another host can leak into this launch, e.g. a Linux
+  // /home/... Python path being used on an Apple Silicon MLX server.
+  const venv = host
+    ? (server?.envPath || typedVenv || '')
+    : (typedVenv || server?.envPath || _envState.envPath || '');
   const label = host
     ? (server?.name ? `${server.name} (${host})` : host)
     : (server?.name || 'local server');
@@ -593,8 +660,8 @@ function _backendChoicesForTarget(target) {
     return [['llamacpp','llama.cpp'],['diffusers','Diffusers']];
   }
   return _isMetal()
-    ? [['llamacpp','llama.cpp'],['ollama','Ollama']]
+    ? [['mlx','MLX'],['llamacpp','llama.cpp'],['ollama','Ollama']]
-    : [['vllm','vLLM'],['sglang','SGLang'],['llamacpp','llama.cpp'],['ollama','Ollama'],['diffusers','Diffusers']];
+    : [['vllm','vLLM'],['sglang','SGLang'],['llamacpp','llama.cpp'],['ollama','Ollama'],['mlx','MLX'],['diffusers','Diffusers']];
 }
 
 async function _fetchServeRuntimePackage(panel, backend) {
@@ -602,6 +669,7 @@ async function _fetchServeRuntimePackage(panel, backend) {
     vllm: 'vllm',
     sglang: 'sglang',
     llamacpp: 'llama_cpp',
+    mlx: 'mlx_lm',
     diffusers: 'diffusers',
   };
   const packageName = packageByBackend[backend];
@@ -621,7 +689,7 @@ async function _fetchServeRuntimePackage(panel, backend) {
 }
 
 function _runtimeNoteText(backend, pkg, target) {
-  const labels = { vllm: 'vLLM', sglang: 'SGLang', llamacpp: 'llama.cpp', diffusers: 'Diffusers' };
+  const labels = { vllm: 'vLLM', sglang: 'SGLang', llamacpp: 'llama.cpp', mlx: 'MLX', diffusers: 'Diffusers' };
   const label = labels[backend] || backend;
   if (!pkg) return `${label} readiness unavailable for ${target.label}.`;
   const note = pkg.status_note || pkg.update_note || '';
@@ -1109,7 +1177,14 @@ function _rerenderCachedModels() {
       const repo = item.dataset.repo;
       if (!repo) return;
       const m = allModels.find(x => x.repo_id === repo);
-      if (!m || m.status !== 'ready') return;
+      if (!m) return;
+      if (m.status !== 'ready') {
+        if (m.status === 'downloading' && !_isActivelyDownloading(m.repo_id)) {
+          uiModule.showToast?.('Refreshing cached model status…');
+          _fetchCachedModels(true);
+        }
+        return;
+      }
 
       // Toggle — close if already open
       if (item.classList.contains('doclib-card-expanded')) {
@@ -1266,7 +1341,7 @@ function _rerenderCachedModels() {
       // stays as the source-of-truth so every existing change handler
       // (updateBackendVisibility, runtime readiness, command builder)
       // still fires via dispatchEvent('change') on selection.
-      panelHtml += `<label>${_l('Engine','Inference engine: vLLM, SGLang, llama.cpp, Ollama, or Diffusers')}<div class="hwfit-backend-picker" data-backend-picker style="position:relative;width:100%;"><select class="hwfit-sf hwfit-backend-source" data-field="backend" style="display:none;">${backendOpts}</select><button type="button" class="hwfit-backend-btn" data-backend-btn aria-haspopup="listbox" aria-expanded="false" style="display:flex;align-items:center;gap:6px;width:100%;height:32px;padding:0 8px;background:var(--bg);color:var(--fg);border:1px solid var(--border);border-radius:4px;font:inherit;font-size:11px;cursor:pointer;text-align:left;position:relative;top:-4px;"><span class="hwfit-backend-btn-icon" data-backend-icon-slot aria-hidden="true" style="display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;color:var(--accent, var(--red));flex-shrink:0;"></span><span class="hwfit-backend-btn-label" data-backend-label style="flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;"></span><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.6;flex-shrink:0;"><polyline points="6 9 12 15 18 9"/></svg></button><div class="hwfit-backend-menu" data-backend-menu role="listbox" hidden style="position:absolute;top:calc(100% + 4px);left:0;right:0;z-index:100;background:var(--panel, var(--bg));border:1px solid var(--border);border-radius:6px;box-shadow:0 6px 20px rgba(0,0,0,0.22);padding:4px;"></div></div></label>`;
+      panelHtml += `<label>${_l('Engine','Inference engine: MLX, vLLM, SGLang, llama.cpp, Ollama, or Diffusers')}<div class="hwfit-backend-picker" data-backend-picker style="position:relative;width:100%;"><select class="hwfit-sf hwfit-backend-source" data-field="backend" style="display:none;">${backendOpts}</select><button type="button" class="hwfit-backend-btn" data-backend-btn aria-haspopup="listbox" aria-expanded="false" style="display:flex;align-items:center;gap:6px;width:100%;height:32px;padding:0 8px;background:var(--bg);color:var(--fg);border:1px solid var(--border);border-radius:4px;font:inherit;font-size:11px;cursor:pointer;text-align:left;position:relative;top:-4px;"><span class="hwfit-backend-btn-icon" data-backend-icon-slot aria-hidden="true" style="display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;color:var(--accent, var(--red));flex-shrink:0;"></span><span class="hwfit-backend-btn-label" data-backend-label style="flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;"></span><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.6;flex-shrink:0;"><polyline points="6 9 12 15 18 9"/></svg></button><div class="hwfit-backend-menu" data-backend-menu role="listbox" hidden style="position:absolute;top:calc(100% + 4px);left:0;right:0;z-index:100;background:var(--panel, var(--bg));border:1px solid var(--border);border-radius:6px;box-shadow:0 6px 20px rgba(0,0,0,0.22);padding:4px;"></div></div></label>`;
       panelHtml += `<input type="hidden" class="hwfit-sf" data-field="host" value="${esc(_es.remoteHost || '')}" />`;
       // Inference mode pill (llama.cpp only) — lives directly to the
       // RIGHT of Backend in Row 1 so the engine and the GPU/CPU choice
@@ -1328,13 +1403,13 @@ function _rerenderCachedModels() {
       // TP / Context / GPU / GPU Mem / Max Seqs / Dtype. Everything else
       // (Swap, KV Cache, Attention backend, Env vars, llama.cpp batch/ubatch)
       // moved to the Advanced fold below to keep this row scannable.
-      panelHtml += `<div class="hwfit-serve-row hwfit-serve-row-core hwfit-backend-vllm hwfit-backend-sglang hwfit-backend-llamacpp hwfit-backend-ollama">`;
+      panelHtml += `<div class="hwfit-serve-row hwfit-serve-row-core hwfit-backend-vllm hwfit-backend-sglang hwfit-backend-llamacpp hwfit-backend-ollama hwfit-backend-mlx">`;
       // Order: Dtype → TP → Context → Max Seqs → GPUs → GPU Mem.
       // Dtype moved down from Row 1 to make space for the Inference pill
       // (llama.cpp GPU/CPU toggle, llamacpp-only). GPUs lives next to
       // GPU Mem so "which devices + how much" sit adjacent. Max Seqs
       // follows Context per the "request-shape" cluster.
-      panelHtml += `<label>${_l('Dtype','Data type for weights. auto picks best for GPU')}<select class="hwfit-sf" data-field="dtype">${dtypeOpts}</select></label>`;
+      panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang hwfit-backend-llamacpp">${_l('Dtype','Data type for weights. auto picks best for GPU')}<select class="hwfit-sf" data-field="dtype">${dtypeOpts}</select></label>`;
       panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang">${_l('TP','Tensor Parallelism — split model across N GPUs')}<select class="hwfit-sf" data-field="tp">${tpOpts}</select></label>`;
       // ctx resets to the model's max on every panel open (the real ctx slider
       // lives in the Scan/Download toolbar — see cookbook.js .hwfit-ctx-control).
@@ -1405,7 +1480,9 @@ function _rerenderCachedModels() {
       panelHtml += `<label>Width${_h('Default output width')} <input type="text" class="hwfit-sf" data-field="diff_width" value="${esc(sv('diff_width', ''))}" placeholder="1024" /></label>`;
       panelHtml += `<label>Height${_h('Default output height')} <input type="text" class="hwfit-sf" data-field="diff_height" value="${esc(sv('diff_height', ''))}" placeholder="1024" /></label>`;
       panelHtml += `</div>`;
-      // Row 3: Checkboxes (vLLM)
+      // Row 3: Advanced toggles for vLLM/SGLang. Several concepts overlap,
+      // but the actual flags differ; keep labels backend-neutral where a
+      // shared checkbox maps to different runtime flags.
       // Order: Trust Remote → Auto Tool → Reasoning Parser (when the
       // model has one) → Enforce Eager → Prefix Caching. Reasoning
       // Parser was previously in a separate row below; the user wanted
@@ -1416,21 +1493,22 @@ function _rerenderCachedModels() {
       const _rp_flag = _opts2_row3.flags.find(f => f.includes('--reasoning-parser'));
       const _rp_name = _rp_flag ? _rp_flag.split(' ')[1] : '';
       panelHtml += `<div class="hwfit-serve-checks hwfit-backend-vllm hwfit-backend-sglang">`;
-      panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="trust_remote"${sv('trust_remote',_isMiniMaxMSeries)?' checked':''} /> Trust Remote Code${_h('Allow model to run custom code from HuggingFace')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="trust_remote"${sv('trust_remote',_isMiniMaxMSeries)?' checked':''} /> Trust Remote Code${_h('SGLang/vLLM: allow model code from HuggingFace via --trust-remote-code')}</label>`;
-      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-backend-sglang"><input type="checkbox" class="hwfit-sf" data-field="auto_tool"${sv('auto_tool',_nativeToolDefault)?' checked':''} /> Auto Tool Choice${_h('Enable function/tool calling for agent mode')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-backend-sglang"><input type="checkbox" class="hwfit-sf" data-field="auto_tool"${sv('auto_tool',_nativeToolDefault)?' checked':''} /> Auto Tool Choice${_h('SGLang/vLLM: enable native tool calling and auto-pick the detected tool-call parser')}</label>`;
       // Always-render the Reasoning Parser, Expert Parallel, and MoE Env
       // checkboxes — the model-family detection above is a hint, not a
       // hard gate. User asked to keep these visible regardless so that
       // a borderline-undetected MoE/reasoning model can still toggle
       // them without dropping back to the raw command box.
-      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-backend-sglang"><input type="checkbox" class="hwfit-sf" data-field="reasoning_parser" data-parser="${_rp_name || ''}"${sv('reasoning_parser',_reasoningDefault)?' checked':''} /> Reasoning Parser${_rp_name ? ` <span class="hwfit-parser-tag">${_rp_name}</span>` : ''}${_h('Splits <think> tokens into a separate channel. The tag (when shown) is the auto-detected parser; edit the command if you need a different one.')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-backend-sglang"><input type="checkbox" class="hwfit-sf" data-field="reasoning_parser" data-parser="${_rp_name || ''}"${sv('reasoning_parser',_reasoningDefault)?' checked':''} /> Reasoning Parser${_rp_name ? ` <span class="hwfit-parser-tag">${_rp_name}</span>` : ''}${_h('SGLang/vLLM: splits thinking tokens into a reasoning channel using the detected parser.')}</label>`;
-      panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="enforce_eager"${sv('enforce_eager',false)?' checked':''} /> Enforce Eager${_h('Disable CUDA graphs. Slower but uses less memory')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="enforce_eager"${sv('enforce_eager',false)?' checked':''} /> Disable CUDA Graphs${_h('vLLM: --enforce-eager. SGLang: --disable-cuda-graph. Slower, but useful for graph-capture crashes.')}</label>`;
-      panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="prefix_cache"${sv('prefix_cache',false)?' checked':''} /> Prefix Caching${_h('Cache shared prompt prefixes across requests')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="prefix_cache"${sv('prefix_cache',false)?' checked':''} /> Prefix / Radix Cache${_h('vLLM: prefix caching. SGLang: RadixAttention prefix cache; when off Odysseus adds --disable-radix-cache.')}</label>`;
       // Inline the previously-second vLLM checks row so Expert Parallel /
       // Speculative / MoE Env sit next to Prefix Caching with no gap. All
       // three are vLLM-only — class-gated so they hide on SGLang. Always
       // render so the user can flip them on for any MoE model.
-      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-backend-sglang"><input type="checkbox" class="hwfit-sf" data-field="expert_parallel"${sv('expert_parallel',_expertParallelDefault)?' checked':''} /> Expert Parallel${_h('MoE: shard expert layers across GPUs. Helps for MiniMax M-series, StepFun Step-3, Qwen3 A3B/A10B/A22B MoE, DeepSeek V3+/R1. Ignored / wasteful on dense models.')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm hwfit-backend-sglang"><input type="checkbox" class="hwfit-sf" data-field="expert_parallel"${sv('expert_parallel',_expertParallelDefault)?' checked':''} /> Expert Parallel${_h('SGLang/vLLM MoE: shard expert layers across GPUs. Useful for DeepSeek/MiniMax/Qwen MoE; avoid on dense models.')}</label>`;
+      panelHtml += `<label class="hwfit-sf-cb hwfit-backend-sglang">Decode Graph${_h('SGLang only: tune decode CUDA graph capture. Smaller batch can fix DeepSeek-V4 graph-capture errors; disabled is safest but slower.')} <select class="hwfit-sf" data-field="sglang_decode_graph" style="height:24px;max-width:92px;"><option value=""${sv('sglang_decode_graph','') === '' ? ' selected' : ''}>auto</option><option value="bs16"${sv('sglang_decode_graph','') === 'bs16' ? ' selected' : ''}>bs 16</option><option value="disabled"${sv('sglang_decode_graph','') === 'disabled' ? ' selected' : ''}>off</option></select></label>`;
       panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm"><input type="checkbox" class="hwfit-sf" data-field="language_model_only"${sv('language_model_only',_isMiniMaxM3)?' checked':''} /> Language Model Only${_h('vLLM --language-model-only. Needed by MiniMax M3 text serving when the repo also contains VL components.')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb hwfit-backend-vllm"><input type="checkbox" class="hwfit-sf" data-field="disable_custom_all_reduce"${sv('disable_custom_all_reduce',_isMiniMaxM3)?' checked':''} /> Disable Custom All Reduce${_h('vLLM --disable-custom-all-reduce. Useful for some 8-GPU/nightly configurations.')}</label>`;
       {
@@ -1585,6 +1663,7 @@ function _rerenderCachedModels() {
         const buildTarget = _selectedServeTarget(panel);
         f.host = buildTarget.host || '';
         f.platform = buildTarget.platform || '';
+        f.venv = buildTarget.venv || '';
         const hostField = panel.querySelector('[data-field="host"]');
         if (hostField) hostField.value = f.host;
         const backend = f.backend || 'vllm';
@@ -1871,6 +1950,7 @@ function _rerenderCachedModels() {
       const _BACKEND_GLYPHS = {
         vllm:   '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"/><path d="M14 4l4 9 3-9"/></svg>',
         sglang: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;"></span>',
+        mlx: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M4 18V6l4 7 4-7v12"/><path d="M16 6v12"/><path d="M20 6v12"/></svg>',
         llamacpp: '<svg width="14" height="14" viewBox="0 0 600 600" fill="none" aria-hidden="true"><path d="M600 392L504.249 558L504.137 557.929C487.252 584.069 458.193 600 426.864 600H120L240 392H600Z" fill="currentColor"/><path d="M240 392H0L199.602 46.0254C216.032 17.5463 246.411 0 279.29 0H466.154L240 392Z" fill="currentColor"/></svg>',
         ollama: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;"></span>',
         diffusers: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"/></svg>',
@@ -1984,7 +2064,7 @@ function _rerenderCachedModels() {
         const backend = panel.querySelector('[data-field="backend"]')?.value || 'vllm';
         const noteText = note.querySelector('.hwfit-serve-runtime-text');
         const _writeNote = (s) => { if (noteText) noteText.textContent = s; else note.textContent = s; };
-        if (!['vllm', 'sglang', 'llamacpp', 'diffusers'].includes(backend)) {
+        if (!['vllm', 'sglang', 'llamacpp', 'mlx', 'diffusers'].includes(backend)) {
           note.style.display = 'none';
           _writeNote('');
           return;
@@ -2024,7 +2104,7 @@ function _rerenderCachedModels() {
             // recipe panel for this backend so the user has one click
             // to the fix instead of hunting for the right row.
             if (noteText) {
-              const pkgName = pkg?.name || ({ vllm: 'vllm', sglang: 'sglang', llamacpp: 'llama_cpp', diffusers: 'diffusers' }[backend]);
+              const pkgName = pkg?.name || ({ vllm: 'vllm', sglang: 'sglang', llamacpp: 'llama_cpp', mlx: 'mlx_lm', diffusers: 'diffusers' }[backend]);
               const repo = (panel.closest('.doclib-card, .memory-item')?.dataset?.repo) || '';
               const link = document.createElement('a');
               link.href = '#';
@@ -2079,7 +2159,7 @@ function _rerenderCachedModels() {
           });
         } else {
           const fields = {
-            backend: cmd.includes('llama_cpp') || cmd.includes('llama-server') ? 'llamacpp' : cmd.includes('diffusion_server') ? 'diffusers' : cmd.includes('sglang') ? 'sglang' : cmd.includes('ollama') ? 'ollama' : 'vllm',
+            backend: cmd.includes('llama_cpp') || cmd.includes('llama-server') ? 'llamacpp' : cmd.includes('mlx_lm.server') ? 'mlx' : cmd.includes('diffusion_server') ? 'diffusers' : cmd.includes('sglang') ? 'sglang' : cmd.includes('ollama') ? 'ollama' : 'vllm',
             port: _ex(/--port\s+(\d+)/) || '8000',
             tp: _ex(/--tensor-parallel-size\s+(\d+)/) || '1',
             ctx: _ex(/--max-model-len\s+(\d+)/) || _ex(/--n_ctx\s+(\d+)/) || _ex(/-c\s+(\d+)/) || '8192',
@@ -3771,7 +3851,8 @@ export async function _fetchCachedModels(fresh = false, opts = {}) {
     const modelDirs = [];
     if (selectedServer && Array.isArray(selectedServer.modelDirs)) {
       for (const d of selectedServer.modelDirs) {
-        if (d && d !== '~/.cache/huggingface/hub') modelDirs.push(d);
+        const normalized = _normalizeCookbookModelDir(d);
+        if (normalized && normalized !== '~/.cache/huggingface/hub') modelDirs.push(normalized);
       }
     }
     // Sync the header dir pills to THIS server (the one whose models we're listing).
@@ -3783,7 +3864,7 @@ export async function _fetchCachedModels(fresh = false, opts = {}) {
       const _allDirs = (Array.isArray(selectedServer.modelDirs) && selectedServer.modelDirs.length
         ? selectedServer.modelDirs
         : [selectedServer.modelDir || '~/.cache/huggingface/hub'])
-        .map(d => (d || '').replaceAll('✕', '').replaceAll('✖', '').trim()).filter(Boolean);
+        .map(d => _normalizeCookbookModelDir(d)).filter(Boolean);
       _dirsEl.innerHTML = _allDirs.map(d => `<span class="cookbook-serve-dir-pill">${esc(d)}</span>`).join('')
         + '<span class="cookbook-serve-dir-edit" title="Edit in Settings">edit</span>';
       _dirsEl.querySelector('.cookbook-serve-dir-edit')?.addEventListener('click', () => {
@@ -3803,10 +3884,12 @@ export async function _fetchCachedModels(fresh = false, opts = {}) {
     }
     if (!allowNetwork) {
       _dlWp.destroy();
-      list.innerHTML = '<div class="hwfit-loading" style="flex-direction:column;gap:8px;text-align:center;"><div>No cached model scan yet</div><div style="font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;">Check this server\'s model cache.</div><button type="button" class="hwfit-gpu-btn serve-empty-scan-btn" style="height:26px;padding:3px 10px;">Scan</button></div>';
+      const wp = spinnerModule.createWhirlpool(22);
-      list.querySelector('.serve-empty-scan-btn')?.addEventListener('click', () => {
+      list.innerHTML = '<div class="hwfit-loading serve-empty-auto-scan" style="flex-direction:column;gap:8px;text-align:center;"><div class="serve-empty-auto-wp"></div><div>No cached model scan yet</div><div style="font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;">Scanning this server\'s model cache…</div></div>';
-        _fetchCachedModels(true);
+      list.querySelector('.serve-empty-auto-wp')?.appendChild(wp.element);
-      });
+      setTimeout(() => {
+        if (list.querySelector('.serve-empty-auto-scan')) _fetchCachedModels(true);
+      }, 60);
       const tagContainer = document.getElementById('serve-tags');
       if (tagContainer) tagContainer.innerHTML = '';
       return;

static/js/document.js

@@ -2288,6 +2288,61 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     return header + '\n---\n' + body;
   }
 
+  function _looksLikeWrappedEmailContent(text) {
+    const t = String(text || '').replace(/\r\n/g, '\n').trim();
+    return /\n---\n/.test(t) && /^(To|Cc|Bcc|Subject|In-Reply-To|References|X-Source-UID|X-Source-Folder):\s*/im.test(t);
+  }
+
+  function _decodeBase64EmailWrapper(block) {
+    const compact = String(block || '').replace(/\s+/g, '');
+    if (compact.length < 24 || compact.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(compact)) return null;
+    try {
+      const bin = atob(compact);
+      let decoded = '';
+      if (typeof TextDecoder !== 'undefined') {
+        const bytes = new Uint8Array(bin.length);
+        for (let i = 0; i < bin.length; i += 1) bytes[i] = bin.charCodeAt(i);
+        decoded = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+      } else {
+        decoded = decodeURIComponent(escape(bin));
+      }
+      decoded = decoded.replace(/\r\n/g, '\n');
+      return _looksLikeWrappedEmailContent(decoded) ? decoded : null;
+    } catch (_) {
+      return null;
+    }
+  }
+
+  function _sanitizeOutgoingEmailBody(raw) {
+    let text = String(raw || '').replace(/\r\n/g, '\n');
+    const trimmed = text.trim();
+    const decodedWhole = _decodeBase64EmailWrapper(trimmed);
+    if (decodedWhole) text = _parseEmailHeader(decodedWhole).body || '';
+    else if (_looksLikeWrappedEmailContent(trimmed)) text = _parseEmailHeader(trimmed).body || '';
+
+    const parts = text.split(/(\n{2,})/);
+    let changed = false;
+    const clean = parts.map(part => {
+      if (/^\n+$/.test(part)) return part;
+      const decoded = _decodeBase64EmailWrapper(part);
+      if (!decoded) return part;
+      changed = true;
+      return _parseEmailHeader(decoded).body || '';
+    }).join('');
+
+    if (!changed && /<[^>]+>/.test(text) && typeof document !== 'undefined') {
+      const probe = document.createElement('div');
+      probe.innerHTML = text;
+      const plain = (probe.innerText || probe.textContent || '').trim();
+      const plainClean = plain ? _sanitizeOutgoingEmailBody(plain) : plain;
+      if (plainClean !== plain) return plainClean;
+    }
+
+    return (changed ? clean : text)
+      .replace(/\n{3,}/g, '\n\n')
+      .trim();
+  }
+
   function _emailLocalDraftKey(sourceUid, sourceFolder, inReplyTo) {
     const uid = String(sourceUid || '').trim();
     if (!uid) return '';
@@ -2328,7 +2383,7 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
       references: draft.references ?? fields.references,
       sourceUid: draft.sourceUid ?? fields.sourceUid,
       sourceFolder: draft.sourceFolder ?? fields.sourceFolder,
-      body: draft.body ?? fields.body,
+      body: _sanitizeOutgoingEmailBody(draft.body ?? fields.body),
     };
   }
 
@@ -2378,20 +2433,27 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
   }
 
   // ── WYSIWYG email body helpers ──
+  function _emailPlainTextToHtml(text) {
+    const d = document.createElement('div');
+    d.textContent = text == null ? '' : String(text);
+    return d.innerHTML.replace(/\n/g, '<br>');
+  }
+
   function _emailBodyToHtml(text) {
     const t = (text || '').trim();
     if (!t) return '';
     // If it already contains a formatting/structural HTML tag, it's a saved
-    // WYSIWYG body — use it verbatim. (Checking a leading '<' isn't enough: a
+    // WYSIWYG body — sanitize it before rendering. (Checking a leading '<' isn't enough: a
     // rich body often starts with plain text, e.g. "Hi <b>there</b>".)
-    if (/<\/?(b|i|u|s|strong|em|del|strike|a|p|div|br|ul|ol|li|h[1-3]|blockquote|span|code|pre)\b[^>]*>/i.test(t)) return t;
+    if (/<\/?(b|i|u|s|strong|em|del|strike|a|p|div|br|ul|ol|li|h[1-3]|blockquote|span|code|pre)\b[^>]*>/i.test(t)) {
+      return markdownModule.sanitizeAllowedHtml
+        ? markdownModule.sanitizeAllowedHtml(t)
+        : _emailPlainTextToHtml(t);
+    }
     // Email body: keep author-typed `:shortcode:` text literal. Issue #345
     // (shortcode → emoji) is scoped to chat; do not rewrite colons in mail.
     try { return markdownModule.mdToHtml(text, { shortcodes: false }); }
-    catch (_) {
+    catch (_) { return _emailPlainTextToHtml(text); }
-      const d = document.createElement('div'); d.textContent = text;
-      return d.innerHTML.replace(/\n/g, '<br>');
-    }
   }
   // Mirror the rich body's plain text into the hidden textarea so the existing
   // send / draft / change-detection plumbing (which reads the textarea) stays
@@ -2629,6 +2691,15 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
   function _splitEmailReplyQuote(text) {
     const original = String(text || '');
     if (!original) return { body: '', quote: '', stripped: false };
+    const literal = '---------- Previous message ----------';
+    const literalIdx = original.indexOf(literal);
+    if (literalIdx >= 0) {
+      return {
+        body: original.slice(0, literalIdx).trim(),
+        quote: original.slice(literalIdx).trim(),
+        stripped: true,
+      };
+    }
     const htmlQuoteOffset = _emailQuoteStartOffset(original);
     if (htmlQuoteOffset >= 0) {
       const body = original.slice(0, htmlQuoteOffset).trim();
@@ -2749,7 +2820,7 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     if (_shouldAutoCollapseEmailHeader()) _setEmailHeaderCollapsed(true, { manual: false });
   }
 
-  function _showEmailFields(doc) {
+  function _showEmailFields(doc, { applyLocalDraft = true } = {}) {
     const emailHeader = document.getElementById('doc-email-header');
     const emailActions = document.getElementById('doc-email-actions');
     // Show MD toolbar for email too (B, I, etc.)
@@ -2782,11 +2853,12 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     document.getElementById('doc-editor-code')?.classList.add('email-mode');
     document.getElementById('doc-editor-highlight')?.classList.add('email-mode');
     let fields = _parseEmailHeader(doc.content || '');
-    fields = _emailFieldsWithLocalDraft(fields);
+    if (applyLocalDraft) fields = _emailFieldsWithLocalDraft(fields);
+    const preserveEmailHeader = !!(fields.sourceUid || fields.inReplyTo || fields.references);
     const subjectInput = document.getElementById('doc-email-subject');
     const textarea = document.getElementById('doc-editor-textarea');
-    _setEmailHeaderInputValue('doc-email-to', fields.to, { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-to', fields.to, { preserveNonEmpty: preserveEmailHeader });
-    _setEmailHeaderInputValue('doc-email-subject', fields.subject, { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-subject', fields.subject, { preserveNonEmpty: preserveEmailHeader });
     _setEmailHeaderCollapsed(!!(doc && doc._emailHeaderCollapsed), { manual: false });
     if (subjectInput && !subjectInput._emailTabBodyBound) {
       subjectInput._emailTabBodyBound = true;
@@ -2797,10 +2869,10 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
         }
       });
     }
-    _setEmailHeaderInputValue('doc-email-in-reply-to', fields.inReplyTo, { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-in-reply-to', fields.inReplyTo, { preserveNonEmpty: preserveEmailHeader });
-    _setEmailHeaderInputValue('doc-email-references', fields.references, { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-references', fields.references, { preserveNonEmpty: preserveEmailHeader });
-    _setEmailHeaderInputValue('doc-email-source-uid', fields.sourceUid || '', { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-source-uid', fields.sourceUid || '', { preserveNonEmpty: preserveEmailHeader });
-    _setEmailHeaderInputValue('doc-email-source-folder', fields.sourceFolder || '', { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-source-folder', fields.sourceFolder || '', { preserveNonEmpty: preserveEmailHeader });
     // Show/hide unread button only if we have a source UID (came from inbox)
     const unreadBtn = document.getElementById('doc-email-unread-btn');
     if (unreadBtn) unreadBtn.style.display = fields.sourceUid ? '' : 'none';
@@ -2923,8 +2995,8 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     const ccRow = document.getElementById('doc-email-cc-row');
     const bccRow = document.getElementById('doc-email-bcc-row');
     const ccToggle = document.getElementById('doc-email-show-cc');
-    _setEmailHeaderInputValue('doc-email-cc', fields.cc || '', { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-cc', fields.cc || '', { preserveNonEmpty: preserveEmailHeader });
-    _setEmailHeaderInputValue('doc-email-bcc', fields.bcc || '', { preserveNonEmpty: true });
+    _setEmailHeaderInputValue('doc-email-bcc', fields.bcc || '', { preserveNonEmpty: preserveEmailHeader });
     const hasCcBcc = !!(
       fields.cc ||
       fields.bcc ||
@@ -3055,6 +3127,292 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     await _uploadComposeFiles(files);
   }
 
+  let _odysseusAttachMenu = null;
+
+  function _closeOdysseusAttachMenu() {
+    if (_odysseusAttachMenu) {
+      _odysseusAttachMenu.remove();
+      _odysseusAttachMenu = null;
+    }
+    document.removeEventListener('click', _attachMenuOutsideClick, true);
+    document.removeEventListener('keydown', _attachMenuEscape, true);
+  }
+
+  function _attachMenuOutsideClick(e) {
+    if (_odysseusAttachMenu && !_odysseusAttachMenu.contains(e.target)) _closeOdysseusAttachMenu();
+  }
+
+  function _attachMenuEscape(e) {
+    if (e.key !== 'Escape') return;
+    _closeOdysseusAttachMenu();
+  }
+
+  function _positionOdysseusAttachMenu(anchor, menu) {
+    const r = anchor?.getBoundingClientRect?.();
+    if (!r) return;
+    menu.style.left = `${Math.max(8, Math.min(r.left, window.innerWidth - 310))}px`;
+    menu.style.top = `${r.bottom + 6}px`;
+    requestAnimationFrame(() => {
+      const mr = menu.getBoundingClientRect();
+      if (mr.bottom > window.innerHeight - 8) {
+        menu.style.top = `${Math.max(8, r.top - mr.height - 6)}px`;
+      }
+    });
+  }
+
+  function _odysseusAttachLabel(item, kind) {
+    if (kind === 'gallery') {
+      return item.caption || item.prompt || item.filename || 'Gallery image';
+    }
+    return item.title || 'Untitled document';
+  }
+
+  async function _stageOdysseusAttachment(kind, id) {
+    const doc = docs.get(activeDocId);
+    if (!doc || doc.language !== 'email') return null;
+    if (!doc._composeAtts) doc._composeAtts = [];
+    const res = await fetch(`${API_BASE}/api/email/compose-from-odysseus`, {
+      method: 'POST',
+      credentials: 'same-origin',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ kind, id }),
+    });
+    let data = null;
+    try { data = await res.json(); } catch (_) {}
+    if (!res.ok || !data?.success) throw new Error(data?.error || data?.detail || `HTTP ${res.status}`);
+    doc._composeAtts.push({
+      token: data.token,
+      filename: data.filename,
+      size: data.size || 0,
+    });
+    return data;
+  }
+
+  async function _stageOdysseusZip(items) {
+    const doc = docs.get(activeDocId);
+    if (!doc || doc.language !== 'email') return null;
+    if (!doc._composeAtts) doc._composeAtts = [];
+    const res = await fetch(`${API_BASE}/api/email/compose-from-odysseus-zip`, {
+      method: 'POST',
+      credentials: 'same-origin',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ items }),
+    });
+    let data = null;
+    try { data = await res.json(); } catch (_) {}
+    if (!res.ok || !data?.success) throw new Error(data?.error || data?.detail || `HTTP ${res.status}`);
+    doc._composeAtts.push({
+      token: data.token,
+      filename: data.filename,
+      size: data.size || 0,
+    });
+    return data;
+  }
+
+  function _afterOdysseusAttachmentsAdded(count, label) {
+    _renderComposeAttachments();
+    clearTimeout(_autoSaveDebounce);
+    _autoSaveDebounce = setTimeout(() => { saveDocument({ silent: true }); }, 800);
+    if (uiModule) uiModule.showToast(count > 1 ? `Attached ${count} items` : `Attached ${label || 'item'}`);
+  }
+
+  async function _attachOdysseusItem(kind, id, label, opts = {}) {
+    try {
+      const data = await _stageOdysseusAttachment(kind, id);
+      if (!data) return;
+      _afterOdysseusAttachmentsAdded(1, label || data.filename);
+      if (!opts.keepOpen) _closeOdysseusAttachMenu();
+    } catch (err) {
+      console.error('Failed to attach Odysseus item:', err);
+      if (uiModule) uiModule.showError('Failed to attach from Odysseus');
+    }
+  }
+
+  function _selectedOdysseusAttachRows(menu) {
+    return Array.from(menu?.querySelectorAll?.('.email-odysseus-attach-row.is-selected') || []);
+  }
+
+  function _syncOdysseusAttachSelection(menu) {
+    const selected = _selectedOdysseusAttachRows(menu);
+    const bar = menu?.querySelector?.('.email-odysseus-attach-actions');
+    const count = menu?.querySelector?.('.email-odysseus-attach-count');
+    const attachBtn = menu?.querySelector?.('.email-odysseus-attach-selected');
+    if (bar) bar.style.display = '';
+    if (count) count.textContent = selected.length ? `${selected.length} selected` : 'Select items to attach';
+    if (attachBtn) attachBtn.disabled = selected.length === 0;
+  }
+
+  async function _attachSelectedOdysseusItems(menu) {
+    const rows = _selectedOdysseusAttachRows(menu);
+    if (!rows.length) return;
+    const btn = menu.querySelector('.email-odysseus-attach-selected');
+    if (btn) {
+      btn.disabled = true;
+      btn.classList.add('is-loading');
+    }
+    let added = 0;
+    try {
+      const items = rows.map(row => ({ kind: row.dataset.kind, id: row.dataset.id })).filter(x => x.kind && x.id);
+      let zip = false;
+      if (items.length > 5) {
+        const ask = window.styledConfirm || uiModule?.styledConfirm;
+        zip = ask
+          ? await ask(`Attach ${items.length} files as one zip?`, { confirmText: 'Zip', cancelText: 'Separate' })
+          : window.confirm(`Attach ${items.length} files as one zip?`);
+      }
+      if (zip) {
+        await _stageOdysseusZip(items);
+        added = 1;
+      } else {
+        for (const item of items) {
+          await _stageOdysseusAttachment(item.kind, item.id);
+          added += 1;
+        }
+      }
+      _afterOdysseusAttachmentsAdded(added, zip ? 'odysseus-attachments.zip' : undefined);
+      _closeOdysseusAttachMenu();
+    } catch (err) {
+      console.error('Failed to attach selected Odysseus items:', err);
+      if (uiModule) uiModule.showError(added ? `Attached ${added}, then failed` : 'Failed to attach from Odysseus');
+      _renderComposeAttachments();
+    } finally {
+      if (btn) {
+        btn.classList.remove('is-loading');
+        btn.disabled = false;
+      }
+    }
+  }
+
+  async function _loadOdysseusAttachItems(menu, kind) {
+    const list = menu.querySelector('.email-odysseus-attach-list');
+    if (!list) return;
+    menu.dataset.odyAttachKind = kind;
+    list.replaceChildren(spinnerModule.createLoadingRow('Loading…', 14));
+    menu.querySelectorAll('[data-ody-attach-kind]').forEach(btn => {
+      btn.classList.toggle('active', btn.dataset.odyAttachKind === kind);
+    });
+    const q = (menu.querySelector('.email-odysseus-attach-search')?.value || '').trim();
+    try {
+      const params = new URLSearchParams({ sort: 'recent', limit: '20' });
+      if (q) params.set('search', q);
+      const endpoint = kind === 'gallery'
+        ? `${API_BASE}/api/gallery/library?${params}`
+        : `${API_BASE}/api/documents/library?${params}`;
+      const res = await fetch(endpoint, { credentials: 'same-origin' });
+      const data = await res.json();
+      if (!res.ok) throw new Error(data?.error || data?.detail || `HTTP ${res.status}`);
+      const items = kind === 'gallery'
+        ? (Array.isArray(data?.items) ? data.items : Array.isArray(data?.images) ? data.images : [])
+        : (Array.isArray(data?.documents) ? data.documents : Array.isArray(data?.items) ? data.items : []);
+      if (!items.length) {
+        list.innerHTML = `<div class="email-odysseus-attach-empty">${q ? 'No matches' : `No ${kind === 'gallery' ? 'images' : 'documents'}`}</div>`;
+        _syncOdysseusAttachSelection(menu);
+        return;
+      }
+      list.innerHTML = '';
+      for (const item of items) {
+        const label = _odysseusAttachLabel(item, kind);
+        const row = document.createElement('button');
+        row.type = 'button';
+        row.className = `email-odysseus-attach-row ${kind === 'gallery' ? 'is-gallery' : ''}`;
+        row.dataset.id = item.id || '';
+        row.dataset.kind = kind;
+        if (kind === 'gallery') {
+          const src = item.url ? `${API_BASE}${item.url}` : '';
+          row.innerHTML = `
+            <span class="email-odysseus-attach-dot" aria-hidden="true"></span>
+            <span class="email-odysseus-attach-thumb">${src ? `<img src="${_escHtml(src)}" alt="">` : ''}</span>
+            <span class="email-odysseus-attach-main">
+              <span class="email-odysseus-attach-title">${_escHtml(label)}</span>
+              <span class="email-odysseus-attach-meta">${_escHtml(item.filename || 'image')}</span>
+            </span>
+          `;
+        } else {
+          row.innerHTML = `
+            <span class="email-odysseus-attach-dot" aria-hidden="true"></span>
+            <span class="email-odysseus-attach-icon">${langIcon(item.language || 'text', 14, { style: 'opacity:0.8;' })}</span>
+            <span class="email-odysseus-attach-main">
+              <span class="email-odysseus-attach-title">${_escHtml(label)}</span>
+              <span class="email-odysseus-attach-meta">${_escHtml(item.language || 'text')}</span>
+            </span>
+          `;
+        }
+        row.addEventListener('click', (ev) => {
+          ev.preventDefault();
+          row.classList.toggle('is-selected');
+          _syncOdysseusAttachSelection(menu);
+        });
+        row.addEventListener('dblclick', () => _attachOdysseusItem(kind, item.id, label, { keepOpen: false }));
+        list.appendChild(row);
+      }
+      _syncOdysseusAttachSelection(menu);
+    } catch (err) {
+      console.error('Failed to load Odysseus attach items:', err);
+      list.innerHTML = '<div class="email-odysseus-attach-empty">Could not load</div>';
+    }
+  }
+
+  function _showComposeAttachMenu(anchor) {
+    if (_activeDocLanguage() !== 'email') {
+      document.getElementById('doc-md-image-input')?.click();
+      return;
+    }
+    _closeOdysseusAttachMenu();
+    const menu = document.createElement('div');
+    menu.className = 'email-odysseus-attach-menu';
+    menu.innerHTML = `
+      <button type="button" class="email-odysseus-attach-local">
+        <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="17 8 12 3 7 8"/><line x1="12" y1="3" x2="12" y2="15"/></svg>
+        Upload file
+      </button>
+      <div class="email-odysseus-attach-tabs">
+        <button type="button" data-ody-attach-kind="document" class="active">
+          <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.1" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/><path d="M14 2v6h6"/><path d="M8 13h8"/><path d="M8 17h6"/></svg>
+          <span>Documents</span>
+        </button>
+        <button type="button" data-ody-attach-kind="gallery">
+          <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.1" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><rect x="3" y="3" width="18" height="18" rx="2"/><circle cx="8.5" cy="8.5" r="1.5"/><path d="M21 15l-5-5L5 21"/></svg>
+          <span>Gallery</span>
+        </button>
+      </div>
+      <label class="email-odysseus-attach-search-wrap">
+        <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="11" cy="11" r="8"/><path d="m21 21-4.35-4.35"/></svg>
+        <input type="search" class="email-odysseus-attach-search" placeholder="Search attachments">
+      </label>
+      <div class="email-odysseus-attach-list"></div>
+      <div class="email-odysseus-attach-actions">
+        <span class="email-odysseus-attach-count"></span>
+        <button type="button" class="email-odysseus-attach-selected" disabled>
+          <svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="m21.44 11.05-9.19 9.19a6 6 0 0 1-8.49-8.49l8.57-8.57A4 4 0 1 1 17.93 8.8l-8.59 8.57a2 2 0 0 1-2.83-2.83l8.49-8.48"></path></svg>
+          <span>Attach</span>
+        </button>
+      </div>
+    `;
+    document.body.appendChild(menu);
+    _odysseusAttachMenu = menu;
+    _positionOdysseusAttachMenu(anchor, menu);
+    menu.querySelector('.email-odysseus-attach-local')?.addEventListener('click', () => {
+      _closeOdysseusAttachMenu();
+      document.getElementById('doc-email-file-input')?.click();
+    });
+    menu.querySelectorAll('[data-ody-attach-kind]').forEach(btn => {
+      btn.addEventListener('click', () => _loadOdysseusAttachItems(menu, btn.dataset.odyAttachKind));
+    });
+    let attachSearchTimer = null;
+    menu.querySelector('.email-odysseus-attach-search')?.addEventListener('input', () => {
+      clearTimeout(attachSearchTimer);
+      attachSearchTimer = setTimeout(() => {
+        _loadOdysseusAttachItems(menu, menu.dataset.odyAttachKind || 'document');
+      }, 220);
+    });
+    menu.querySelector('.email-odysseus-attach-selected')?.addEventListener('click', () => _attachSelectedOdysseusItems(menu));
+    setTimeout(() => {
+      document.addEventListener('click', _attachMenuOutsideClick, true);
+      document.addEventListener('keydown', _attachMenuEscape, true);
+    }, 0);
+    _loadOdysseusAttachItems(menu, 'document');
+  }
+
   function _isMarkdownImageFile(file) {
     if (!file) return false;
     if ((file.type || '').toLowerCase().startsWith('image/')) return true;
@@ -3241,13 +3599,23 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
         }).filter(Boolean)
       );
       sugg.innerHTML = '';
+      sugg.dataset.navStarted = '0';
       let count = 0;
       for (const c of data.results) {
         for (const em of (c.emails || [])) {
           if (already.has(em.toLowerCase())) continue;
           const item = document.createElement('div');
           item.className = 'contact-suggestion';
+          item.setAttribute('role', 'option');
+          item.setAttribute('aria-selected', 'false');
           item.innerHTML = `<span class="contact-name">${_escHtml(c.name)}</span><span class="contact-email">${_escHtml(em)}</span>`;
+          item.addEventListener('mouseenter', () => {
+            sugg.dataset.navStarted = '1';
+            sugg.querySelectorAll('.contact-suggestion').forEach(it => {
+              it.classList.toggle('active', it === item);
+              it.setAttribute('aria-selected', it === item ? 'true' : 'false');
+            });
+          });
           // mousedown fires before blur so the click doesn't get lost
           item.addEventListener('mousedown', (e) => { e.preventDefault(); _commitRecipient(input, sugg, em); });
           item.addEventListener('click', (e) => { e.preventDefault(); _commitRecipient(input, sugg, em); });
@@ -3256,9 +3624,6 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
         }
       }
       if (count === 0) { sugg.style.display = 'none'; return; }
-      // Auto-highlight first suggestion so Enter accepts it.
-      const first = sugg.querySelector('.contact-suggestion');
-      if (first) first.classList.add('active');
       sugg.style.display = '';
     } catch (e) {
       sugg.style.display = 'none';
@@ -3284,16 +3649,32 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
       const items = open ? sugg.querySelectorAll('.contact-suggestion') : [];
       const active = open ? sugg.querySelector('.contact-suggestion.active') : null;
       let idx = active ? Array.from(items).indexOf(active) : -1;
+      const setActive = (nextIdx) => {
+        items.forEach((it, i) => {
+          const on = i === nextIdx;
+          it.classList.toggle('active', on);
+          it.setAttribute('aria-selected', on ? 'true' : 'false');
+        });
+        if (items[nextIdx]) {
+          items[nextIdx].scrollIntoView({ block: 'nearest' });
+        }
+      };
       if (open && e.key === 'ArrowDown') {
         e.preventDefault();
-        idx = Math.min(items.length - 1, idx + 1);
+        if (!items.length) return;
-        items.forEach(it => it.classList.remove('active'));
+        if (sugg.dataset.navStarted !== '1') {
-        if (items[idx]) items[idx].classList.add('active');
+          idx = Math.max(0, idx);
+          sugg.dataset.navStarted = '1';
+        } else {
+          idx = Math.min(items.length - 1, idx + 1);
+        }
+        setActive(idx);
       } else if (open && e.key === 'ArrowUp') {
         e.preventDefault();
+        if (!items.length) return;
+        sugg.dataset.navStarted = '1';
         idx = Math.max(0, idx - 1);
-        items.forEach(it => it.classList.remove('active'));
+        setActive(idx);
-        if (items[idx]) items[idx].classList.add('active');
       } else if (e.key === 'Enter') {
         // If a suggestion is highlighted, commit it. Otherwise — if the
         // current fragment already looks like a complete email — commit
@@ -3410,8 +3791,10 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     const _rich = _emailRichbodyActive();
     if (_rich) _syncEmailRichbody(_rich);
     const textarea = document.getElementById('doc-editor-textarea');
-    const body = (_rich ? (_rich.innerText || _rich.textContent || '') : (textarea?.value || '')).trim();
+    const rawBody = (_rich ? (_rich.innerText || _rich.textContent || '') : (textarea?.value || '')).trim();
-    const bodyHtml = _rich ? _rich.innerHTML : null;
+    const body = _sanitizeOutgoingEmailBody(rawBody);
+    let bodyHtml = _rich ? _rich.innerHTML : null;
+    if (_rich && body !== rawBody) bodyHtml = _emailBodyToHtml(body);
     const doc = docs.get(activeDocId);
     const attachments = (doc?._composeAtts || []).map(a => a.token);
     if (!to || !body) {
@@ -3574,8 +3957,10 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     const _rich = _emailRichbodyActive();
     if (_rich) _syncEmailRichbody(_rich);
     const textarea = document.getElementById('doc-editor-textarea');
-    const body = (_rich ? (_rich.innerText || _rich.textContent || '') : (textarea?.value || '')).trim();
+    const rawBody = (_rich ? (_rich.innerText || _rich.textContent || '') : (textarea?.value || '')).trim();
-    const bodyHtml = _rich ? _rich.innerHTML : null;
+    const body = _sanitizeOutgoingEmailBody(rawBody);
+    let bodyHtml = _rich ? _rich.innerHTML : null;
+    if (_rich && body !== rawBody) bodyHtml = _emailBodyToHtml(body);
     const btn = document.getElementById('doc-email-draft-btn');
     if (btn) { btn.disabled = true; btn.textContent = 'Saving...'; }
     const controller = new AbortController();
@@ -3917,10 +4302,11 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     const references = document.getElementById('doc-email-references')?.value?.trim();
     const _rich = _emailRichbodyActive();
     if (_rich) _syncEmailRichbody(_rich);
-    const body = (_rich
+    const rawBody = (_rich
       ? (_rich.innerText || _rich.textContent || '')
       : (document.getElementById('doc-editor-textarea')?.value || '')
     ).trim();
+    const body = _sanitizeOutgoingEmailBody(rawBody);
     const doc = docs.get(activeDocId);
     const attachments = (doc?._composeAtts || []).map(a => a.token);
 
@@ -5139,12 +5525,12 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     }, true);
 
     // Attachments
-    document.getElementById('doc-email-attach-btn')?.addEventListener('click', () => {
+    document.getElementById('doc-email-attach-btn')?.addEventListener('click', (e) => {
-      document.getElementById('doc-email-file-input')?.click();
+      _showComposeAttachMenu(e.currentTarget);
     });
-    document.getElementById('md-toolbar-attach-btn')?.addEventListener('click', () => {
+    document.getElementById('md-toolbar-attach-btn')?.addEventListener('click', (e) => {
       if (_activeDocLanguage() === 'email') {
-        document.getElementById('doc-email-file-input')?.click();
+        _showComposeAttachMenu(e.currentTarget);
       } else {
         document.getElementById('doc-md-image-input')?.click();
       }
@@ -10056,11 +10442,33 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     if (!docs.has(docId)) {
       const curSession = sessionModule?.getCurrentSessionId() || '';
       let reuseId = null;
+      const incomingFields = _parseEmailHeader(newContent || '');
+
+      // Email subjects repeat constantly ("test", "Re: ..."). Match open
+      // compose docs by source email identity first; never let a same-title
+      // draft steal an update meant for a different open email.
+      if (incomingFields.sourceUid) {
+        const wantFolder = (incomingFields.sourceFolder || 'INBOX').trim();
+        for (const [existingId, existingDoc] of docs) {
+          const existingFields = _parseEmailHeader(existingDoc.content || '');
+          if (
+            String(existingFields.sourceUid || '') === String(incomingFields.sourceUid)
+            && ((existingFields.sourceFolder || 'INBOX').trim() === wantFolder)
+          ) {
+            reuseId = existingId;
+            break;
+          }
+        }
+      }
 
       // First: match by title
-      if (data.title) {
+      if (!reuseId && data.title) {
         for (const [existingId, existingDoc] of docs) {
-          if (existingDoc.title === data.title && existingDoc.sessionId === curSession) {
+          if (
+            existingDoc.title === data.title
+            && existingDoc.sessionId === curSession
+            && (existingDoc.language || '').toLowerCase() !== 'email'
+          ) {
             reuseId = existingId;
             break;
           }
@@ -10195,8 +10603,10 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
       if (isEmailUpdate) {
         const updatedDocForEmail = docs.get(docId);
         if (updatedDocForEmail) {
+          const updatedFields = _parseEmailHeader(updatedDocForEmail.content || '');
+          _clearEmailLocalDraft(updatedFields.sourceUid, updatedFields.sourceFolder, updatedFields.inReplyTo);
           _setMarkdownPreviewActive(false, { remember: false });
-          _showEmailFields(updatedDocForEmail);
+          _showEmailFields(updatedDocForEmail, { applyLocalDraft: false });
         }
       } else {
         if (textarea) textarea.value = newContent;
@@ -10219,7 +10629,9 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
     if (isEmailUpdate && updatedDoc) {
       updatedDoc.language = 'email';
       if (langSelect) langSelect.value = 'email';
-      _showEmailFields(updatedDoc);
+      const updatedFields = _parseEmailHeader(updatedDoc.content || '');
+      _clearEmailLocalDraft(updatedFields.sourceUid, updatedFields.sourceFolder, updatedFields.inReplyTo);
+      _showEmailFields(updatedDoc, { applyLocalDraft: false });
     }
     if (updatedDoc && !updatedDoc.userSetLanguage && !updatedDoc.language) {
       setTimeout(attemptAutoDetect, 100);

static/js/emailInbox.js

@@ -8,7 +8,7 @@ import sessionModule from './sessions.js';
 import { initEmailLibrary, openEmailLibrary, closeEmailLibrary, isOpen as isLibOpen } from './emailLibrary.js';
 import * as Modals from './modalManager.js';
 import { applyEdgeDock } from './modalSnap.js';
-import { buildReplyAllCc } from './emailLibrary/replyRecipients.js';
+import { buildReplyAllCc, extractEmail } from './emailLibrary/replyRecipients.js';
 import { emailApiUrl, emailAccountQuery } from './emailShared.js';
 import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
 
@@ -29,6 +29,23 @@ const _icon = (svg) => `<span class="dropdown-icon">${svg}</span>`;
 const _replySeparator = '---------- Previous message ----------';
 const _DONE_RESPONSE_TAGS = new Set(['urgent', 'reply-soon', 'action-needed']);
 
+function _splitEmailAddresses(raw) {
+  return (typeof raw === 'string' ? raw : '')
+    .split(',')
+    .map((x) => x.trim())
+    .filter(Boolean);
+}
+
+function _isMyEmailAddress(addr, myAddresses) {
+  const email = extractEmail(addr);
+  if (!email) return false;
+  return new Set((myAddresses || []).map(a => String(a || '').trim().toLowerCase()).filter(Boolean)).has(email);
+}
+
+function _withoutMyAddresses(raw, myAddresses) {
+  return _splitEmailAddresses(raw).filter(addr => !_isMyEmailAddress(addr, myAddresses));
+}
+
 function _openCalendarEventFromEmail(uid) {
   const target = String(uid || '').trim();
   if (!target) return;
@@ -832,13 +849,26 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
       ? window._myEmailAddresses
       : (window._myEmailAddress ? [window._myEmailAddress] : []);
 
-    let toAddress = data.from_address;
+    const fromIsMe = _isMyEmailAddress(data.from_address, myAddresses);
+    const originalToWithoutMe = _withoutMyAddresses(data.to, myAddresses);
+    const originalCcWithoutMe = _withoutMyAddresses(data.cc, myAddresses);
+
+    let toAddress = fromIsMe
+      ? (originalToWithoutMe.join(', ') || originalCcWithoutMe[0] || data.from_address)
+      : data.from_address;
     let ccAddresses = '';
     let subjectPrefix = 'Re: ';
 
     if (mode === 'reply-all') {
-      // Build reply-all: TO = original sender, CC = everyone else (To + Cc minus me)
+      if (fromIsMe) {
-      ccAddresses = buildReplyAllCc(data, myAddresses);
+        // Replying from Sent should go back to the people I originally wrote
+        // to, not to myself. Keep original Cc recipients on Cc.
+        toAddress = originalToWithoutMe.join(', ') || originalCcWithoutMe[0] || data.from_address;
+        ccAddresses = originalCcWithoutMe.filter(addr => !originalToWithoutMe.some(t => extractEmail(t) === extractEmail(addr))).join(', ');
+      } else {
+        // Build reply-all: TO = original sender, CC = everyone else (To + Cc minus me)
+        ccAddresses = buildReplyAllCc(data, myAddresses);
+      }
     } else if (mode === 'forward') {
       toAddress = '';
       subjectPrefix = 'Fwd: ';
@@ -921,7 +951,7 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
       // Agent-provided reply text should land in the email draft the user
       // already has open. Otherwise mobile users see the source email while the
       // agent silently creates a second draft elsewhere.
-      const reuseExisting = (mode === 'view' || mode === 'open' || (!!aiSuggestedBody && mode !== 'forward'));
+      const reuseExisting = mode !== 'forward';
       const existingDocId = (reuseExisting && _docModule.findEmailDocId)
         ? _docModule.findEmailDocId(em.uid, _currentFolder)
         : null;
@@ -930,52 +960,22 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
         await new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)));
         await _docModule.loadDocument(existingDocId);
         if (aiSuggestedBody && typeof _docModule.replaceEmailReplyBody === 'function') {
-          await _docModule.replaceEmailReplyBody(existingDocId, aiSuggestedBody);
+          await _docModule.replaceEmailReplyBody(existingDocId, aiSuggestedBody, { force: true });
-          _bringEmailReplyDraftToFrontOnMobile();
         }
+        _bringEmailReplyDraftToFrontOnMobile();
       } else {
-        // If the user already has a chat session open, reuse it instead of
+        const activeSid = await _createEmailChat(data);
-        // spawning a new one. They asked for this explicitly — opening reply
-        // mid-conversation shouldn't whip them out of context.
-        let activeSid = '';
-        try { activeSid = sessionModule?.getCurrentSessionId?.() || ''; } catch {}
         if (!activeSid) {
-          // No chat in flight — keep the old behavior of creating a scoped
+          console.error('reply: could not obtain a session_id');
-          // email-thread chat, then RE-READ the now-current session id. The
+          import('./ui.js').then(m => m.showError && m.showError('Could not start a reply chat.')).catch(() => {});
-          // POST below requires a session_id (backend 400s without one), and
+          return;
-          // the freshly-created chat is what should own the reply draft.
-          await _createEmailChat(data);
-          try { activeSid = sessionModule?.getCurrentSessionId?.() || ''; } catch {}
-        }
-        // Guarantee a session — _createEmailChat can't make one when there's
-        // no enabled default-chat endpoint, which left the reply POSTing a
-        // null session_id → 400. Create a bare session so the draft always
-        // has a home regardless of chat/endpoint config.
-        if (!activeSid) {
-          try {
-            const _fd = new FormData();
-            _fd.append('name', `Email: ${(data.subject || '').slice(0, 60)}`);
-            _fd.append('skip_validation', 'true');
-            const _sres = await fetch(`${API_BASE}/api/session`, { method: 'POST', body: _fd, credentials: 'same-origin' });
-            if (_sres.ok) {
-              const _sdata = await _sres.json();
-              if (_sdata && _sdata.id) {
-                activeSid = _sdata.id;
-                if (sessionModule?.loadSessions) await sessionModule.loadSessions();
-                if (sessionModule?.selectSession) await sessionModule.selectSession(activeSid);
-              }
-            }
-          } catch (e) { console.error('reply: bare session create failed', e); }
         }
 
         const docRes = await fetch(`${API_BASE}/api/document`, {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify({
-            // Reuse the user's current chat session if there is one (so the
+            session_id: activeSid,
-            // reply draft lives in the chat they were just in); otherwise
-            // null and the new email-chat (created above) takes over.
-            session_id: activeSid || null,
             title: data.subject,
             content: content,
             language: 'email',
@@ -1256,31 +1256,58 @@ async function _toggleDone(em, itemEl) {
 }
 
 async function _createEmailChat(emailData) {
+  const subject = String(emailData?.subject || 'New Email').trim() || 'New Email';
+  const title = subject === 'New Email' ? 'New Email' : `Email: ${subject.slice(0, 60)}`;
   try {
-    // Try current session's endpoint first
+    const currentSid = sessionModule.getCurrentSessionId?.() || '';
-    const current = sessionModule.getSessions?.().find(s => s.id === sessionModule.getCurrentSessionId?.());
+    const current = sessionModule.getSessions?.().find(s => s.id === currentSid);
-    let url, model, endpointId;
+    const currentIsBlank = !!current
-    if (current && current.endpoint_url && current.model) {
+      && !current.archived
-      url = current.endpoint_url;
+      && !current.has_documents
-      model = current.model;
+      && !current.has_images
-      endpointId = current.endpoint_id;
+      && Number(current.message_count || 0) === 0
-    } else {
+      && current.folder !== 'Assistant'
-      // Fall back to default chat config
+      && current.folder !== 'Tasks';
-      const dcRes = await fetch(`${API_BASE}/api/default-chat`);
+    if (currentIsBlank) {
-      const dc = await dcRes.json();
+      const meta = document.getElementById('current-meta');
-      url = dc.endpoint_url;
+      if (meta) meta.textContent = title;
-      model = dc.model;
+      return current.id;
-      endpointId = dc.endpoint_id;
+    }
+    let url = current?.endpoint_url || '';
+    let model = current?.model || '';
+    let endpointId = current?.endpoint_id || '';
+    if (!url || !model) {
+      try {
+        const dcRes = await fetch(`${API_BASE}/api/default-chat`, { credentials: 'same-origin' });
+        const dc = dcRes.ok ? await dcRes.json() : {};
+        url = dc.endpoint_url || '';
+        model = dc.model || '';
+        endpointId = dc.endpoint_id || '';
+      } catch (_) {}
     }
 
-    if (url && model) {
+    const fd = new FormData();
-      await sessionModule.createDirectChat(url, model, endpointId);
+    fd.append('name', title);
-      // Set a helpful title in the chat meta
+    fd.append('skip_validation', 'true');
-      const meta = document.getElementById('current-meta');
+    if (url) fd.append('endpoint_url', url);
-      if (meta) meta.textContent = `Email: ${(emailData.subject || '').slice(0, 60)}`;
+    if (model) fd.append('model', model);
+    if (endpointId) fd.append('endpoint_id', endpointId);
+    const res = await fetch(`${API_BASE}/api/session`, { method: 'POST', body: fd, credentials: 'same-origin' });
+    if (!res.ok) {
+      console.error('email chat create failed', res.status, await res.text().catch(() => ''));
+      return '';
     }
+    const payload = await res.json().catch(() => ({}));
+    const sid = payload?.id || '';
+    if (!sid) return '';
+    if (sessionModule?.loadSessions) await sessionModule.loadSessions();
+    if (sessionModule?.selectSession) await sessionModule.selectSession(sid);
+    const meta = document.getElementById('current-meta');
+    if (meta) meta.textContent = title;
+    return sid;
   } catch (e) {
     console.error('Failed to create email chat:', e);
+    return '';
   }
 }
 
@@ -1292,38 +1319,7 @@ async function _composeNew() {
   // (doc shows for a frame, then slides up again). Mount once, at injectFreshDoc,
   // after the session + doc exist.
   try {
-    // /api/document requires a session_id (returns 400 if null), so reuse
+    const sid = await _createEmailChat({ subject: 'New Email' });
-    // the active chat if there is one — otherwise spin up an email-scoped
-    // chat first, same pattern the reply path uses.
-    let sid = '';
-    try { sid = sessionModule?.getCurrentSessionId?.() || ''; } catch (_) {}
-    if (!sid) {
-      await _createEmailChat({ subject: 'New Email' });
-      try { sid = sessionModule?.getCurrentSessionId?.() || ''; } catch (_) {}
-    }
-    // Guarantee a session — _createEmailChat can't make one when there's no
-    // enabled default-chat endpoint, which left compose POSTing a null
-    // session_id → 400 (the draft silently never appeared). Same bare-session
-    // fallback the reply flow uses.
-    if (!sid) {
-      try {
-        const _fd = new FormData();
-        _fd.append('name', 'New Email');
-        _fd.append('skip_validation', 'true');
-        const _sres = await fetch(`${API_BASE}/api/session`, { method: 'POST', body: _fd, credentials: 'same-origin' });
-        if (_sres.ok) {
-          const _sdata = await _sres.json();
-          if (_sdata && _sdata.id) {
-            sid = _sdata.id;
-            // NOTE: intentionally do NOT loadSessions()/selectSession() here.
-            // Re-selecting the (empty) session re-renders the chat and flashes
-            // the welcome splash for a frame before the draft opens — the
-            // "splash flickers like crazy then email opens" bug. The doc only
-            // needs the session_id; the draft opens in the doc panel regardless.
-          }
-        }
-      } catch (e) { console.error('compose: bare session create failed', e); }
-    }
     if (!sid) {
       console.error('compose: could not obtain a session_id');
       import('./ui.js').then(m => m.showError && m.showError('Could not start a new email (no session).')).catch(() => {});

static/js/emailLibrary.js

@@ -35,6 +35,8 @@ let _libSearchSeq = 0;
 let _libSearchHadResults = false;
 let _libSearchInFlight = false;
 let _activeEmailReaderForSelectAll = null;
+let _libAccountsLoadedAt = 0;
+const _LIB_ACCOUNTS_TTL_MS = 5 * 60 * 1000;
 
 function _isEmailTypingTarget(t) {
   return !!(t && (
@@ -950,8 +952,20 @@ function _libCachePut(key, value) {
   }
 }
 
+function _resetBulkSelectionForContextChange({ rerender = false } = {}) {
+  const hadSelection = !!(state._selectedUids && state._selectedUids.size);
+  const wasSelectMode = !!state._selectMode;
+  if (state._selectedUids) state._selectedUids.clear();
+  state._selectMode = false;
+  if (hadSelection || wasSelectMode) {
+    _updateBulkBar();
+    if (rerender) _renderGrid();
+  }
+}
+
 function _resetEmailListForFreshLoad() {
   _exitEmailReaderModeForList();
+  _resetBulkSelectionForContextChange();
   state._libOffset = 0;
   state._libEmails = [];
   state._libTotal = 0;
@@ -981,6 +995,28 @@ function _loadEmailsFresh() {
   return _loadEmails({ force: true, useCache: false });
 }
 
+function _isChatInteractionBusy() {
+  try {
+    if (window.__odysseusChatBusy) return true;
+    const until = Number(window.__odysseusChatBusyUntil || 0);
+    return until > Date.now();
+  } catch (_) {
+    return false;
+  }
+}
+
+function _loadEmailsWhenChatIdle({ delay = 700, retries = 180, options = {} } = {}) {
+  const run = () => {
+    if (!state._libOpen || !document.getElementById('email-lib-modal')) return;
+    if (_isChatInteractionBusy() && retries > 0) {
+      setTimeout(() => _loadEmailsWhenChatIdle({ delay: 1000, retries: retries - 1, options }), 1000);
+      return;
+    }
+    _loadEmails(options);
+  };
+  setTimeout(run, Math.max(0, Number(delay) || 0));
+}
+
 export function prewarmEmailLibrary({ delay = 2500 } = {}) {
   if (_libPrewarmTimer || _libPrewarmPromise) return;
   const elapsed = Date.now() - _libLastPrewarmAt;
@@ -1011,7 +1047,10 @@ async function _prewarmEmailViews() {
     const accountsRes = await fetch(`${API_BASE}/api/email/accounts`, { credentials: 'same-origin' });
     if (accountsRes.ok) {
       const accountsData = await accountsRes.json().catch(() => ({}));
-      if (Array.isArray(accountsData.accounts)) state._libAccounts = accountsData.accounts;
+      if (Array.isArray(accountsData.accounts)) {
+        state._libAccounts = accountsData.accounts;
+        _libAccountsLoadedAt = Date.now();
+      }
     }
   } catch (_) {}
 
@@ -1737,7 +1776,13 @@ export function openEmailLibrary(opts = {}) {
   };
   document.addEventListener('keydown', state._libEscHandler, true);
 
-  _renderAccountsLoading();
+  const grid = document.getElementById('email-lib-grid');
+  if (grid && !grid.children.length) _renderEmailLoading(grid);
+  if (Array.isArray(state._libAccounts) && state._libAccounts.length) {
+    _renderAccountsStrip();
+  } else {
+    _renderAccountsLoading();
+  }
   // Await accounts before loading emails so the list request carries the
   // right account_id from the very first fetch (now that we auto-select
   // an explicit account instead of relying on a 'Default' chip).
@@ -1745,17 +1790,31 @@ export function openEmailLibrary(opts = {}) {
     await _loadAccounts();
     _loadFolders();
     _loadEmailReminderBellVisibility();
-    _loadEmails();
+    _loadEmailsWhenChatIdle();
   })();
 }
 
-async function _loadAccounts() {
+async function _loadAccounts({ force = false } = {}) {
+  const hasCachedAccounts = Array.isArray(state._libAccounts) && state._libAccounts.length;
+  const accountsFresh = _libAccountsLoadedAt && (Date.now() - _libAccountsLoadedAt) < _LIB_ACCOUNTS_TTL_MS;
+  if (!force && hasCachedAccounts && accountsFresh) {
+    if (!state._libAccountId) {
+      const def = state._libAccounts.find(a => a.is_default) || state._libAccounts[0];
+      state._libAccountId = def?.id || null;
+      _publishActiveAccount();
+    }
+    _renderAccountsStrip();
+    return;
+  }
   try {
     const r = await fetch(`${API_BASE}/api/email/accounts`, { credentials: 'same-origin' });
     if (!r.ok) return;
     const d = await r.json();
     state._libAccounts = d.accounts || [];
-  } catch (_) { state._libAccounts = []; }
+    _libAccountsLoadedAt = Date.now();
+  } catch (_) {
+    if (!hasCachedAccounts) state._libAccounts = [];
+  }
   // The 'Default' chip is gone — pick an explicit account so the email
   // list and any per-email actions (open in new tab, mark read, etc.)
   // always carry an account_id and can't desync from the server's
@@ -2080,6 +2139,60 @@ function _crossFolderCandidates() {
   return Array.from(new Set(candidates.filter(Boolean)));
 }
 
+function _findEmailFolder(patterns, fallback) {
+  const available = Array.isArray(state._libFolders) ? state._libFolders.filter(Boolean) : [];
+  const lower = new Map(available.map(f => [String(f).toLowerCase(), f]));
+  for (const p of patterns) {
+    const direct = lower.get(String(p).toLowerCase());
+    if (direct) return direct;
+  }
+  return available.find(f => patterns.some(p => String(f).toLowerCase().includes(String(p).toLowerCase()))) || fallback;
+}
+
+function _sentFolderName() {
+  return _findEmailFolder(['[Gmail]/Sent Mail', 'Sent Mail', 'Sent Items', 'INBOX.Sent', 'Sent'], 'Sent');
+}
+
+function _deriveSearchScope(rawQuery) {
+  const original = String(rawQuery || '').trim();
+  const tokens = original.split(/\s+/).filter(Boolean);
+  let scope = 'all';
+  const kept = [];
+  let forced = '';
+  for (const token of tokens) {
+    const t = token.toLowerCase().replace(/^#+/, '').replace(/:$/, '');
+    if (['sent', 'sentmail', 'sent-mail', 'outbox'].includes(t)) {
+      forced = 'sent';
+      continue;
+    }
+    if (['inbox'].includes(t)) {
+      forced = 'inbox';
+      continue;
+    }
+    kept.push(token);
+  }
+  if (forced) scope = forced;
+  let folder = 'INBOX';
+  let serverScope = 'all';
+  if (scope === 'sent') {
+    folder = _sentFolderName();
+    serverScope = 'folder';
+  } else if (scope === 'inbox') {
+    folder = 'INBOX';
+    serverScope = 'folder';
+  } else if (scope === 'current') {
+    folder = state._libFolder || 'INBOX';
+    serverScope = 'folder';
+  }
+  return {
+    scope,
+    folder,
+    serverScope,
+    q: forced ? kept.join(' ').trim() : original,
+    forced,
+  };
+}
+
 // Snapshot of state._libEmails taken right before search starts so we
 // can both filter locally and restore on clear without re-fetching.
 let _libPreSearchEmails = null;
@@ -2406,6 +2519,7 @@ function _clearFilterPillSideEffect() {
 
 function _addSearchPill(pill) {
   if (!pill) return;
+  _resetBulkSelectionForContextChange({ rerender: true });
   if (!Array.isArray(state._libSearchPills)) state._libSearchPills = [];
   // Dedup by email (contact), text (text pill), or filter value.
   if (pill.type === 'contact') {
@@ -2429,8 +2543,18 @@ function _addSearchPill(pill) {
   _applyPillFilter();
 }
 
+function _searchQueryFromPills() {
+  const parts = [];
+  for (const p of state._libSearchPills || []) {
+    if (p.type === 'text' && p.text) parts.push(String(p.text).trim());
+    else if (p.type === 'contact' && (p.email || p.name)) parts.push(String(p.email || p.name).trim());
+  }
+  return parts.filter(Boolean).join(' ').trim();
+}
+
 function _removeSearchPillAt(idx) {
   if (!Array.isArray(state._libSearchPills)) return;
+  _resetBulkSelectionForContextChange({ rerender: true });
   const removed = state._libSearchPills[idx];
   state._libSearchPills.splice(idx, 1);
   if (removed && removed.type === 'filter') _clearFilterPillSideEffect();
@@ -2455,6 +2579,26 @@ function _removeSearchPillAt(idx) {
     _loadEmails({ useCache: true });
     return;
   }
+  const remainingQuery = _searchQueryFromPills();
+  if (remainingQuery.length >= 2) {
+    state._libSearch = remainingQuery;
+    const _searchInput = document.getElementById('email-lib-search');
+    if (_searchInput) _searchInput.value = '';
+    state._libSearchDraft = '';
+    _doSearch();
+    return;
+  }
+  if ((state._libSearchPills || []).length && _libSearchHadResults) {
+    _libSearchHadResults = false;
+    _libPreSearchEmails = null;
+    _libPreSearchTotal = 0;
+    _libServerSearchEmails = null;
+    _libServerSearchTotal = 0;
+    state._libSearch = '';
+    state._libOffset = 0;
+    _loadEmails({ useCache: true });
+    return;
+  }
   _applyPillFilter();
 }
 
@@ -2588,6 +2732,7 @@ async function _initEmailSearchChipBar() {
   // directly.
   let _libSearchTypingTimer = null;
   input.addEventListener('input', async () => {
+    _resetBulkSelectionForContextChange({ rerender: true });
     state._libSearchDraft = input.value;
     await _refreshSuggestions();
     if (_libSearchTypingTimer) clearTimeout(_libSearchTypingTimer);
@@ -2723,9 +2868,11 @@ window.addEventListener('click', (e) => {
 
 async function _doSearch() {
   _exitEmailReaderModeForList();
+  _resetBulkSelectionForContextChange({ rerender: true });
   const seq = ++_libSearchSeq;
-  const q = state._libSearch.trim();
+  const derived = _deriveSearchScope(state._libSearch);
-  if (q.length < 2) {
+  const q = derived.q;
+  if (q.length < 2 && !derived.forced) {
     // Empty or too short — restore the normal folder if a previous search
     // had replaced the grid contents.
     if (_libSearchHadResults) {
@@ -2738,7 +2885,8 @@ async function _doSearch() {
     return;
   }
   const accountAtStart = state._libAccountId || '';
-  const folderAtStart = state._libFolder || 'INBOX';
+  const folderAtStart = derived.folder || state._libFolder || 'INBOX';
+  const serverScopeAtStart = derived.serverScope || 'all';
   // No grid-blanking spinner — the local filter already painted something
   // useful. Surface progress in the stats badge instead so the user knows
   // the server search is still grinding.
@@ -2753,25 +2901,67 @@ async function _doSearch() {
 
   const stillCurrent = () => (
     seq === _libSearchSeq &&
-    q === state._libSearch.trim() &&
+    q === _deriveSearchScope(state._libSearch).q &&
     accountAtStart === (state._libAccountId || '') &&
-    folderAtStart === (state._libFolder || 'INBOX')
+    folderAtStart === (_deriveSearchScope(state._libSearch).folder || state._libFolder || 'INBOX')
   );
   const searchUrl = (localOnly = false) => {
     const params = new URLSearchParams({
       folder: folderAtStart,
       q,
       limit: '100',
+      scope: serverScopeAtStart,
     });
     if (accountAtStart) params.set('account_id', accountAtStart);
     if (localOnly) params.set('local_only', '1');
     return `${API_BASE}/api/email/search?${params.toString()}`;
   };
+  const folderListUrl = () => {
+    const params = new URLSearchParams({
+      folder: folderAtStart,
+      limit: '100',
+      offset: '0',
+      filter: state._libFilter || 'all',
+    });
+    if (accountAtStart) params.set('account_id', accountAtStart);
+    return `${API_BASE}/api/email/list?${params.toString()}`;
+  };
+  const mergeSearchResults = (painted, incoming) => {
+    const byKey = new Map();
+    const out = [];
+    const add = (em) => {
+      if (!em) return;
+      const key = `${em.account_id || accountAtStart || ''}:${em.folder || folderAtStart || ''}:${em.uid || em.message_id || JSON.stringify(em)}`;
+      if (byKey.has(key)) return;
+      byKey.set(key, em);
+      out.push(em);
+    };
+    (painted || []).forEach(add);
+    const additions = [];
+    const addIncoming = (em) => {
+      if (!em) return;
+      const key = `${em.account_id || accountAtStart || ''}:${em.folder || folderAtStart || ''}:${em.uid || em.message_id || JSON.stringify(em)}`;
+      if (byKey.has(key)) return;
+      byKey.set(key, em);
+      additions.push(em);
+    };
+    (incoming || []).forEach(addIncoming);
+    additions.sort((a, b) => {
+      const ad = Number(a?.date_epoch || 0);
+      const bd = Number(b?.date_epoch || 0);
+      if (bd !== ad) return bd - ad;
+      return String(b?.date || '').localeCompare(String(a?.date || ''));
+    });
+    return out.concat(additions);
+  };
   let paintedInterimResults = false;
   const paintSearchData = (data, interim = false) => {
     if (!stillCurrent()) return false;
     if (data.error) throw new Error(data.error);
-    const results = data.emails || [];
+    let results = data.emails || [];
+    if (!interim && paintedInterimResults) {
+      results = mergeSearchResults(state._libEmails || [], results);
+    }
     if (!interim && paintedInterimResults && results.length === 0) {
       if (stats) {
         const count = state._libTotal || (state._libEmails || []).length;
@@ -2789,7 +2979,7 @@ async function _doSearch() {
     const preservingBase = !!(_libServerSearchEmails && pills.length > 1);
     if (!preservingBase) {
       _libServerSearchEmails = results.slice();
-      _libServerSearchTotal = data.total || results.length;
+      _libServerSearchTotal = Math.max(Number(data.total || 0), results.length);
       _libPreSearchEmails = results.slice();
       _libPreSearchTotal = _libServerSearchTotal;
       state._libEmails = results;
@@ -2803,7 +2993,7 @@ async function _doSearch() {
       if (!(state._libEmails || []).length && !preservingBase) state._libEmails = results;
     }
     _renderGrid();
-    const count = data.total || results.length;
+    const count = Math.max(Number(data.total || 0), results.length);
     if (stats) {
       if (interim) {
         stats.textContent = `${count} cached match${count === 1 ? '' : 'es'} · searching…`;
@@ -2822,9 +3012,22 @@ async function _doSearch() {
   };
 
   try {
+    if (q.length < 2 && derived.forced) {
+      const res = await fetch(folderListUrl());
+      const data = await res.json();
+      if (!stillCurrent()) return;
+      paintSearchData({
+        emails: (data.emails || []).map(em => ({ ...em, folder: folderAtStart })),
+        total: data.total || (data.emails || []).length,
+        source: 'folder',
+        sync: { source: 'folder' },
+      }, false);
+      return;
+    }
+    const fullSearchPromise = fetch(searchUrl(false)).then(res => res.json());
+    const localSearchPromise = fetch(searchUrl(true)).then(res => res.json());
     try {
-      const localRes = await fetch(searchUrl(true));
+      const localData = await localSearchPromise;
-      const localData = await localRes.json();
       if (!stillCurrent()) return;
       if (!localData.error && (localData.emails || []).length) {
         paintSearchData(localData, true);
@@ -2833,8 +3036,7 @@ async function _doSearch() {
       if (!stillCurrent()) return;
     }
 
-    const res = await fetch(searchUrl(false));
+    const data = await fullSearchPromise;
-    const data = await res.json();
     if (!stillCurrent()) return;
     paintSearchData(data, false);
   } catch (e) {
@@ -3401,9 +3603,12 @@ function _createCard(em) {
     card.appendChild(cb);
   }
 
-  // In Sent folder, show the recipient(s) — the sender is always you and
+  // In Sent results, show the recipient(s) — the sender is always you and
-  // hides the actually useful info. Outside Sent, show the sender as before.
+  // hides the actually useful info. Search results can be stamped with their
-  const isSentFolderEarly = /sent/i.test(state._libFolder);
+  // real folder while the visible folder selector still says INBOX, so use the
+  // email's folder first.
+  const cardFolder = em.folder || state._libFolder || 'INBOX';
+  const isSentFolderEarly = /sent/i.test(cardFolder);
   let senderName;
   let senderAddress;
   if (isSentFolderEarly) {
@@ -3482,7 +3687,7 @@ function _createCard(em) {
   }
 
   // Done check + unread dot stay next to the subject on the left.
-  const isSentFolder = /sent/i.test(state._libFolder);
+  const isSentFolder = /sent/i.test(cardFolder);
   if (!isSentFolder) {
     const doneCheck = document.createElement('span');
     doneCheck.className = 'email-card-done' + (em.is_answered ? ' active' : '');
@@ -3512,10 +3717,10 @@ function _createCard(em) {
       }
       try {
         if (newState) {
-          await fetch(`${API_BASE}/api/email/mark-answered/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' });
+          await fetch(`${API_BASE}/api/email/mark-answered/${em.uid}?folder=${encodeURIComponent(cardFolder)}${_acct()}`, { method: 'POST' });
-          await fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' });
+          await fetch(`${API_BASE}/api/email/mark-read/${em.uid}?folder=${encodeURIComponent(cardFolder)}${_acct()}`, { method: 'POST' });
         } else {
-          await fetch(`${API_BASE}/api/email/clear-answered/${em.uid}?folder=${encodeURIComponent(state._libFolder)}${_acct()}`, { method: 'POST' });
+          await fetch(`${API_BASE}/api/email/clear-answered/${em.uid}?folder=${encodeURIComponent(cardFolder)}${_acct()}`, { method: 'POST' });
         }
       } catch (err) { console.error(err); }
     };
@@ -3571,8 +3776,14 @@ function _createCard(em) {
   const meta = document.createElement('div');
   meta.className = 'memory-item-meta';
   meta.style.cssText = 'font-size:10px;opacity:0.7;margin-top:2px;';
+  const showFolderChip = !!(_libSearchHadResults && cardFolder);
+  const prettyFolder = folderDisplayName(cardFolder);
+  const sentChip = isSentFolderEarly ? '<span class="email-sent-chip" title="Sent email">Sent</span>' : '';
+  const folderChip = showFolderChip && !isSentFolderEarly
+    ? `<span class="email-folder-chip" title="${_esc(cardFolder)}">${_esc(prettyFolder)}</span>`
+    : '';
   const senderPrefix = isSentFolderEarly ? 'to ' : '';
-  meta.innerHTML = `<span class="email-meta-sender" data-email="${_esc(senderAddress || '')}" data-name="${_esc(senderName || '')}"><span style="opacity:0.55">${senderPrefix}</span><span style="color:${color};font-weight:600">${_esc(senderName)}</span></span><span class="email-meta-sep"> · </span><span class="email-meta-date">${_esc(dateStr)}</span>`;
+  meta.innerHTML = `${sentChip}${folderChip}<span class="email-meta-sender" data-email="${_esc(senderAddress || '')}" data-name="${_esc(senderName || '')}"><span style="opacity:0.55">${senderPrefix}</span><span style="color:${color};font-weight:600">${_esc(senderName)}</span></span><span class="email-meta-sep"> · </span><span class="email-meta-date">${_esc(dateStr)}</span>`;
   content.appendChild(meta);
 
   card.appendChild(content);
@@ -5284,6 +5495,63 @@ function _wireAttachmentHandlers(reader, folder) {
   // a ReferenceError when this fn is called from contexts that don't have
   // _isMobileUA in scope (e.g. _openEmailAsTab, _openEmailWindow).
   const _isMobileUA = /Android|iPhone|iPad|iPod/i.test(navigator.userAgent);
+  reader.querySelectorAll('.email-attachments-download-all').forEach(btn => {
+    if (btn.dataset.wired === '1') return;
+    btn.dataset.wired = '1';
+    btn.addEventListener('click', async (ev) => {
+      ev.stopPropagation();
+      ev.preventDefault();
+      if (btn.dataset.downloading === '1') return;
+      const uid = btn.dataset.attUid;
+      const sourceFolder = btn.dataset.attFolder || useFolder;
+      const count = Number(btn.dataset.attCount || 0);
+      if (!uid) return;
+      const originalHtml = btn.innerHTML;
+      const originalTitle = btn.title;
+      btn.dataset.downloading = '1';
+      btn.classList.add('is-loading');
+      try {
+        const sp = window.spinnerModule || (await import('./spinner.js')).default;
+        const wp = sp.createWhirlpool(12);
+        wp.element.style.margin = '0';
+        btn.textContent = '';
+        btn.appendChild(wp.element);
+        const label = document.createElement('span');
+        label.textContent = 'All';
+        btn.appendChild(label);
+      } catch (_) {
+        btn.textContent = 'All...';
+      }
+      try {
+        const url = `${API_BASE}/api/email/attachments-download/${encodeURIComponent(uid)}?folder=${encodeURIComponent(sourceFolder)}${_acct()}`;
+        const res = await fetch(url, { credentials: 'same-origin' });
+        if (!res.ok) {
+          const msg = await res.text().catch(() => '');
+          console.error('attachments zip download failed', res.status, msg);
+          location.href = url;
+          return;
+        }
+        const blob = await res.blob();
+        const blobUrl = URL.createObjectURL(blob);
+        const a = document.createElement('a');
+        a.href = blobUrl;
+        a.download = `email-${uid}-attachments.zip`;
+        document.body.appendChild(a);
+        a.click();
+        a.remove();
+        setTimeout(() => URL.revokeObjectURL(blobUrl), 1000);
+        try { uiModule.showToast && uiModule.showToast(`Downloading ${count || 'all'} attachments`); } catch (_) {}
+      } catch (e) {
+        console.error('attachments zip download error', e);
+        try { const { showError } = await import('./ui.js'); showError('Could not download attachments'); } catch (_) {}
+      } finally {
+        delete btn.dataset.downloading;
+        btn.classList.remove('is-loading');
+        btn.title = originalTitle;
+        btn.innerHTML = originalHtml;
+      }
+    });
+  });
   reader.querySelectorAll('.email-attachment-open').forEach(openBtn => {
     if (openBtn.dataset.wired === '1') return;
     openBtn.dataset.wired = '1';
@@ -5487,11 +5755,15 @@ function _buildAttsHtmlFor(uid, data) {
       ? `Thread attachments (${related.length})`
       : `Hidden inline attachments (${hidden.length})`;
   const startCollapsed = !visible.length && !related.length;
+  const downloadAllBtn = visible.length > 4
+    ? `<button type="button" class="email-attachments-download-all" title="Download all attachments" data-att-uid="${_esc(uid)}" data-att-folder="${_esc(data.folder || state._libFolder || 'INBOX')}" data-att-count="${visible.length}"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"/><polyline points="7 10 12 15 17 10"/><line x1="12" y1="15" x2="12" y2="3"/></svg><span>All</span></button>`
+    : '';
   return (
     `<div class="email-reader-atts-wrap${startCollapsed ? ' collapsed' : ''}">`
     +   '<div class="email-reader-atts-header email-summary-toggle" role="button" tabindex="0">'
     +     '<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m21.44 11.05-9.19 9.19a6 6 0 0 1-8.49-8.49l8.57-8.57A4 4 0 1 1 17.93 8.8l-8.59 8.57a2 2 0 0 1-2.83-2.83l8.49-8.48"/></svg>'
     +     `<span>${label}</span>`
+    +     downloadAllBtn
     +     '<svg class="email-summary-chevron" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-left:auto;transition:transform .15s ease;"><polyline points="6 9 12 15 18 9"/></svg>'
     +   '</div>'
     +   visibleSection
@@ -6350,26 +6622,7 @@ async function _translateEmail(reader, language, opts = {}) {
 }
 
 async function _maybeAutoTranslateEmail(reader) {
-  if (!reader || reader.dataset.autoTranslateChecked === '1') return;
+  if (reader) reader.dataset.autoTranslateChecked = '1';
-  reader.dataset.autoTranslateChecked = '1';
-  try {
-    const res = await fetch(`${API_BASE}/api/email/config`);
-    const cfg = await res.json();
-    if (!cfg || !cfg.email_auto_translate) return;
-    try {
-      const sid = window.sessionModule?.getCurrentSessionId?.() || '';
-      if (sid) {
-        const ctrl = new AbortController();
-        const timer = setTimeout(() => ctrl.abort(), 800);
-        const statusRes = await fetch(`${API_BASE}/api/chat/stream_status/${encodeURIComponent(sid)}`, {
-          signal: ctrl.signal,
-        }).catch(() => null);
-        clearTimeout(timer);
-        if (statusRes && statusRes.ok) return;
-      }
-    } catch (_) {}
-    await _translateEmail(reader, cfg.email_translate_language || 'English', { auto: true });
-  } catch (_) {}
 }
 
 // Keep an email ⋮ dropdown inside the viewport: when it would spill past the

static/js/fileHandler.js

@@ -24,6 +24,131 @@ const MAX_FILES = 10;
 const MAX_VISIBLE = 3;
 let _expanded = false;
 
+function _isMobileViewport() {
+  return window.matchMedia && window.matchMedia('(max-width: 768px)').matches;
+}
+
+function _isCroppableImage(f) {
+  const mime = (f?.type || '').toLowerCase();
+  const name = (f?.name || '').toLowerCase();
+  if (!(mime.startsWith('image/') || /\.(png|jpe?g|webp|bmp)$/i.test(name))) return false;
+  return !mime.includes('svg') && !mime.includes('gif') && !/\.svg|\.gif$/i.test(name);
+}
+
+function _loadImage(url) {
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = () => resolve(img);
+    img.onerror = reject;
+    img.src = url;
+  });
+}
+
+function _canvasToBlob(canvas, type, quality) {
+  return new Promise((resolve) => canvas.toBlob(resolve, type || 'image/png', quality));
+}
+
+async function _openMobileCropper(file) {
+  const url = _getPreviewUrl(file);
+  const imgProbe = await _loadImage(url);
+  return new Promise((resolve) => {
+    const overlay = document.createElement('div');
+    overlay.className = 'attach-crop-overlay';
+    overlay.innerHTML = `
+      <div class="attach-crop-panel" role="dialog" aria-modal="true" aria-label="Crop image">
+        <div class="attach-crop-stage">
+          <img class="attach-crop-img" alt="">
+          <div class="attach-crop-box"><span class="attach-crop-handle"></span></div>
+        </div>
+        <div class="attach-crop-actions">
+          <button type="button" class="attach-crop-btn" data-action="cancel">Cancel</button>
+          <button type="button" class="attach-crop-btn" data-action="original">Original</button>
+          <button type="button" class="attach-crop-btn attach-crop-primary" data-action="crop">Use crop</button>
+        </div>
+      </div>`;
+    document.body.appendChild(overlay);
+    const img = overlay.querySelector('.attach-crop-img');
+    const box = overlay.querySelector('.attach-crop-box');
+    img.src = url;
+    img.alt = file.name || 'image';
+
+    let crop = { x: 0.08, y: 0.08, w: 0.84, h: 0.84 };
+    let drag = null;
+
+    function applyCrop() {
+      const r = img.getBoundingClientRect();
+      const pr = overlay.querySelector('.attach-crop-stage').getBoundingClientRect();
+      box.style.left = (r.left - pr.left + crop.x * r.width) + 'px';
+      box.style.top = (r.top - pr.top + crop.y * r.height) + 'px';
+      box.style.width = (crop.w * r.width) + 'px';
+      box.style.height = (crop.h * r.height) + 'px';
+    }
+    function clampCrop() {
+      crop.w = Math.max(0.12, Math.min(1, crop.w));
+      crop.h = Math.max(0.12, Math.min(1, crop.h));
+      crop.x = Math.max(0, Math.min(1 - crop.w, crop.x));
+      crop.y = Math.max(0, Math.min(1 - crop.h, crop.y));
+    }
+    function finish(value) {
+      overlay.remove();
+      window.removeEventListener('resize', applyCrop);
+      resolve(value);
+    }
+    requestAnimationFrame(applyCrop);
+    img.addEventListener('load', applyCrop);
+    window.addEventListener('resize', applyCrop);
+
+    box.addEventListener('pointerdown', (e) => {
+      e.preventDefault();
+      box.setPointerCapture(e.pointerId);
+      drag = {
+        mode: e.target.classList.contains('attach-crop-handle') ? 'resize' : 'move',
+        sx: e.clientX,
+        sy: e.clientY,
+        start: { ...crop },
+      };
+    });
+    box.addEventListener('pointermove', (e) => {
+      if (!drag) return;
+      const r = img.getBoundingClientRect();
+      const dx = (e.clientX - drag.sx) / Math.max(1, r.width);
+      const dy = (e.clientY - drag.sy) / Math.max(1, r.height);
+      if (drag.mode === 'resize') {
+        crop.w = drag.start.w + dx;
+        crop.h = drag.start.h + dy;
+      } else {
+        crop.x = drag.start.x + dx;
+        crop.y = drag.start.y + dy;
+      }
+      clampCrop();
+      applyCrop();
+    });
+    box.addEventListener('pointerup', () => { drag = null; });
+    box.addEventListener('pointercancel', () => { drag = null; });
+
+    overlay.querySelector('[data-action="cancel"]').addEventListener('click', () => finish(null));
+    overlay.querySelector('[data-action="original"]').addEventListener('click', () => finish(file));
+    overlay.querySelector('[data-action="crop"]').addEventListener('click', async () => {
+      clampCrop();
+      const canvas = document.createElement('canvas');
+      const sx = Math.round(crop.x * imgProbe.naturalWidth);
+      const sy = Math.round(crop.y * imgProbe.naturalHeight);
+      const sw = Math.max(1, Math.round(crop.w * imgProbe.naturalWidth));
+      const sh = Math.max(1, Math.round(crop.h * imgProbe.naturalHeight));
+      canvas.width = sw;
+      canvas.height = sh;
+      const ctx = canvas.getContext('2d');
+      ctx.drawImage(imgProbe, sx, sy, sw, sh, 0, 0, sw, sh);
+      const type = file.type && file.type !== 'image/bmp' ? file.type : 'image/png';
+      const blob = await _canvasToBlob(canvas, type, 0.92);
+      if (!blob) { finish(file); return; }
+      const ext = type.includes('jpeg') ? 'jpg' : (type.split('/')[1] || 'png');
+      const base = (file.name || 'image').replace(/\.[^.]+$/, '');
+      finish(new File([blob], `${base}-cropped.${ext}`, { type, lastModified: Date.now() }));
+    });
+  });
+}
+
 function _getPreviewUrl(f) {
   if (!f) return '';
   let url = _previewUrls.get(f);
@@ -231,17 +356,35 @@ export async function uploadPending(opts = {}) {
 /**
  * Add files to pending list (capped at MAX_FILES)
  */
-export function addFiles(files) {
+export async function addFiles(files) {
   for (const f of files) {
     if (pendingFiles.length >= MAX_FILES) {
       _showToast(`Max ${MAX_FILES} files allowed`);
       break;
     }
-    pendingFiles.push(f);
+    let nextFile = f;
+    if (_isMobileViewport() && _isCroppableImage(f)) {
+      try {
+        nextFile = await _openMobileCropper(f);
+      } catch (_) {
+        nextFile = f;
+      }
+      if (!nextFile) continue;
+    }
+    pendingFiles.push(nextFile);
   }
   renderAttachStrip();
 }
 
+export async function cropForMobileUpload(file) {
+  if (!_isMobileViewport() || !_isCroppableImage(file)) return file;
+  try {
+    return await _openMobileCropper(file);
+  } catch (_) {
+    return file;
+  }
+}
+
 function _showToast(msg) {
   if (window.showToast) { window.showToast(msg); return; }
   // Fallback inline toast
@@ -326,6 +469,7 @@ const fileHandlerModule = {
   removePending,
   uploadPending,
   addFiles,
+  cropForMobileUpload,
   getPendingCount,
   getPendingInfo,
   getPendingRaw,