Bump the python group across 1 directory with 2 updates

Updates the requirements on [httpcore](https://github.com/encode/httpcore) and [pydantic-settings](https://github.com/pydantic/pydantic-settings) to permit the latest version.

Updates `httpcore` to 1.0.9
- [Release notes](https://github.com/encode/httpcore/releases)
- [Changelog](https://github.com/encode/httpcore/blob/master/CHANGELOG.md)
- [Commits](https://github.com/encode/httpcore/compare/1.0.0...1.0.9)

Updates `pydantic-settings` to 2.14.2
- [Release notes](https://github.com/pydantic/pydantic-settings/releases)
- [Commits](https://github.com/pydantic/pydantic-settings/compare/v2.14.1...v2.14.2)

---
updated-dependencies:
- dependency-name: httpcore
  dependency-version: 1.0.9
  dependency-type: direct:production
  dependency-group: python
- dependency-name: pydantic-settings
  dependency-version: 2.14.2
  dependency-type: direct:production
  dependency-group: python
...

Signed-off-by: dependabot[bot] <support@github.com>

.env.example

@@ -169,6 +169,26 @@ SEARXNG_INSTANCE=http://localhost:8080
 # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400                  # speech-to-text audio (25 MB)
 # ODYSSEUS_ICS_MAX_BYTES=10485760                        # calendar .ics import (10 MB)
 
+# ============================================================
+# Host Docker access (explicit opt-in)
+# ============================================================
+# Default Docker Compose does not mount /var/run/docker.sock. Existing
+# Ollama, vLLM, and other OpenAI-compatible endpoints remain usable without it.
+#
+# Enable this only for intentional Cookbook/local Docker-daemon management.
+# Raw socket access is high-trust and can grant broad control over the host
+# Docker daemon. Set DOCKER_GID to the host docker group's numeric GID.
+# Put these values in .env, or export them before running docker compose.
+# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
+# DOCKER_GID=963
+# docker/host-docker.yml sets this inside the container. Keep it paired
+# with the socket overlay; setting it alone is not sufficient.
+# ODYSSEUS_ENABLE_HOST_DOCKER=true
+#
+# Host Docker access can be combined with one GPU overlay:
+# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
+# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
+
 # ============================================================
 # GPU support (Docker Compose)
 # ============================================================

ACKNOWLEDGMENTS.md

@@ -86,6 +86,7 @@ Bundled in `static/fonts/`:
 | [Fira Code](https://github.com/tonsky/FiraCode) | SIL Open Font License 1.1 | Nikita Prokopov & contributors |
 | [Inter](https://github.com/rsms/inter) | SIL Open Font License 1.1 | Rasmus Andersson |
 | [GohuFont](https://font.gohu.org/) (`fonts/custom/GohuFont.ttf`) | WTFPL | Hugo Chargois |
+| [OpenDyslexic](https://opendyslexic.org/) (`fonts/OpenDyslexic-{Regular,Bold}.woff2`) | SIL Open Font License 1.1 ([`licenses/OpenDyslexic-OFL.txt`](licenses/OpenDyslexic-OFL.txt)) | Abbie Gonzalez |
 
 ## Python dependencies
 

Dockerfile

@@ -1,3 +1,14 @@
+# ---- builder: patch + build wheels for Real-ESRGAN's broken-on-3.14 deps ----
+# basicsr/gfpgan/facexlib read their version via exec()+locals()['__version__'],
+# which raises KeyError on Python 3.13+ (PEP 667). Build patched wheels here so
+# the final image / Cookbook never has to compile the broken sdists. See
+# docker/build-realesrgan-wheels.sh for the full rationale.
+FROM python:3.14-slim AS realesrgan-wheels
+RUN apt-get update && apt-get install -y --no-install-recommends curl \
+    && rm -rf /var/lib/apt/lists/*
+COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh
+RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels
+
 FROM python:3.14-slim
 
 # System deps. tmux is required by Cookbook for background downloads/serves.
@@ -18,8 +29,44 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     tmux \
     openssh-client \
     gosu \
+    libgl1 \
+    libglib2.0-0t64 \
+    libxcb1 \
+    libmagic1 \
     && rm -rf /var/lib/apt/lists/*
 
+# libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1,
+# libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The
+# slim base omits them, so the Cookbook "install realesrgan" path imports cv2
+# and dies with `libxcb.so.1: cannot open shared object file` despite a clean
+# pip install. Using full opencv-python (not -headless) because basicsr/gfpgan/
+# facexlib/realesrgan all depend on the `opencv-python` distribution by name.
+#
+# libmagic1 is the shared lib (libmagic.so.1) that python-magic dlopens for
+# content-based MIME sniffing in src/upload_handler.py. We install both here
+# (libmagic1 + the python-magic wrapper, below) rather than in requirements.txt
+# because python-magic resolves libmagic at import time: where the lib is
+# absent the import can block or raise, so keeping it image-only avoids
+# regressing pip/venv installs on hosts without libmagic. Debian always has the
+# lib here, so the import is instant and detection actually works.
+
+# Docker CLI (client only — daemon stays on the host via the
+# /var/run/docker.sock mount). The Debian `docker.io` package ships
+# dockerd but not the client binary on slim, so grab the static client
+# tarball from download.docker.com instead.
+ARG DOCKER_CLI_VERSION=27.5.1
+RUN ARCH="$(dpkg --print-architecture)" \
+    && case "$ARCH" in \
+         amd64) DARCH=x86_64 ;; \
+         arm64) DARCH=aarch64 ;; \
+         *) echo "unsupported arch $ARCH"; exit 1 ;; \
+       esac \
+    && curl -fsSL "https://download.docker.com/linux/static/stable/${DARCH}/docker-${DOCKER_CLI_VERSION}.tgz" \
+       -o /tmp/docker.tgz \
+    && tar -xzf /tmp/docker.tgz -C /tmp \
+    && install -m 0755 /tmp/docker/docker /usr/local/bin/docker \
+    && rm -rf /tmp/docker /tmp/docker.tgz
+
 WORKDIR /app
 
 # Install Python deps first (layer cache). Optional extras (PyMuPDF AGPL, etc.)
@@ -29,6 +76,20 @@ COPY requirements.txt requirements-optional.txt ./
 RUN pip install --no-cache-dir -r requirements.txt \
     && if [ "$INSTALL_OPTIONAL" = "true" ]; then pip install --no-cache-dir -r requirements-optional.txt; fi
 
+# python-magic powers content-based MIME sniffing in src/upload_handler.py.
+# Image-only (not in requirements.txt) because it needs the libmagic1 system
+# lib installed above; see the apt note near the top of this stage.
+RUN pip install --no-cache-dir python-magic==0.4.27
+
+# Pre-install the patched basicsr/gfpgan/facexlib wheels built in the
+# realesrgan-wheels stage (--no-deps keeps the image lean — torch & friends are
+# pulled only when realesrgan is actually installed). With these dists already
+# satisfied, the Cookbook's plain `pip install realesrgan` resolves them from
+# wheels instead of rebuilding the sdists that fail on Python 3.14.
+COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/
+RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \
+    && rm -rf /tmp/odysseus-wheels
+
 # Copy app code
 COPY . .
 

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="280">
+  <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238">
 </p>
 
 <p align="center">
@@ -18,7 +18,7 @@
 </p>
 
 <p align="center">
-  <img src="docs/odysseus.jpg" alt="Odysseus interface">
+  <img src="docs/odysseus-browser.jpg" alt="Odysseus interface">
 </p>
 
 ---

app.py

@@ -2,6 +2,16 @@
 import mimetypes
 import os
 import sys
+import asyncio
+
+# On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
+# When started via `python -m uvicorn` from a terminal, uvicorn sets this
+# automatically. But the VS Code debugger (and other non-uvicorn entrypoints)
+# use the default SelectorEventLoop, which raises NotImplementedError on any
+# subprocess call. Force ProactorEventLoop here so the right loop is always
+# used, regardless of how the process is launched.
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
 
 
 def register_static_mime_types() -> None:
@@ -44,7 +54,7 @@ from typing import Dict
 
 from contextlib import asynccontextmanager
 from fastapi import FastAPI, Request, HTTPException
-from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
+from fastapi.responses import JSONResponse, FileResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from starlette.middleware.base import BaseHTTPMiddleware
@@ -65,7 +75,7 @@ from core.exceptions import (
 
 import bcrypt as _bcrypt
 
-from src.app_helpers import abs_join
+from src.app_helpers import abs_join, serve_html_with_nonce
 from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_image_path
 from starlette.responses import RedirectResponse
 
@@ -187,7 +197,19 @@ class _RequestTimeoutMiddleware(_BaseHTTPMiddleware):
             )
 
 
+class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
+    async def dispatch(self, request, call_next):
+        from src.interactive_gate import should_track_interactive_request, track_interactive_request
+
+        path = request.url.path or ""
+        if not should_track_interactive_request(path, request.method):
+            return await call_next(request)
+        async with track_interactive_request(path, request.method):
+            return await call_next(request)
+
+
 app.add_middleware(_RequestTimeoutMiddleware)
+app.add_middleware(_InteractiveActivityMiddleware)
 
 # ========= AUTH =========
 from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
@@ -573,6 +595,14 @@ webhook_manager = WebhookManager(api_key_manager=api_key_manager)
 auth_router = setup_auth_routes(auth_manager)
 app.include_router(auth_router)
 
+
+@app.post("/api/activity/heartbeat")
+async def activity_heartbeat():
+    from src.interactive_gate import mark_browser_activity
+    await mark_browser_activity()
+    return {"ok": True}
+
+
 # Uploads
 from routes.upload_routes import setup_upload_routes
 upload_router, upload_cleanup_func = setup_upload_routes(upload_handler)
@@ -594,7 +624,7 @@ from routes.admin_wipe_routes import setup_admin_wipe_routes
 app.include_router(setup_admin_wipe_routes(session_manager))
 
 # Memory
-from routes.memory_routes import setup_memory_routes
+from routes.memory.memory_routes import setup_memory_routes
 memory_router = setup_memory_routes(memory_manager, session_manager, memory_vector=memory_vector)
 app.include_router(memory_router)
 from routes.skills_routes import setup_skills_routes
@@ -611,7 +641,7 @@ app.include_router(setup_chat_routes(
 ))
 
 # Research (background deep-research tasks)
-from routes.research_routes import setup_research_routes
+from routes.research.research_routes import setup_research_routes
 app.include_router(setup_research_routes(research_handler, session_manager=session_manager))
 
 # History
@@ -675,7 +705,7 @@ from routes.signature_routes import setup_signature_routes
 app.include_router(setup_signature_routes())
 
 # Gallery (image library)
-from routes.gallery_routes import setup_gallery_routes
+from routes.gallery.gallery_routes import setup_gallery_routes
 app.include_router(setup_gallery_routes())
 
 # Persisted image-editor drafts (server-backed projects)
@@ -791,23 +821,17 @@ app.include_router(setup_companion_routes())
 
 # ========= ROUTES (kept in app.py) =========
 
-def _serve_html_with_nonce(request: Request, file_path: str) -> HTMLResponse:
-    """Read an HTML file and inject the CSP nonce into inline <script> tags."""
-    with open(file_path, "r", encoding="utf-8") as f:
-        html = f.read()
-    nonce = getattr(request.state, "csp_nonce", "")
-    html = html.replace("{{CSP_NONCE}}", nonce)
-    return HTMLResponse(html)
-
 @app.get("/")
 async def serve_index(request: Request):
     static_path = abs_join(BASE_DIR, "static/index.html")
     if os.path.exists(static_path):
-        return _serve_html_with_nonce(request, static_path)
-    root_path = abs_join(BASE_DIR, "index.html")
-    if os.path.exists(root_path):
-        return _serve_html_with_nonce(request, root_path)
-    raise HTTPException(404, "index.html not found")
+        return serve_html_with_nonce(request, static_path)
+    # No static bundle — fall back to a root-level index.html if one is shipped.
+    # If neither exists, serve_html_with_nonce logs it and returns a generic 500:
+    # a missing index.html is a broken deployment (server fault), not a client
+    # "not found". This keeps the app-shell route consistent with the other
+    # bundled-template routes instead of mislabelling the fault as a 404.
+    return serve_html_with_nonce(request, abs_join(BASE_DIR, "index.html"))
 
 @app.get("/notes")
 async def serve_notes(request: Request):
@@ -848,13 +872,13 @@ async def serve_library(request: Request):
 @app.get("/backgrounds")
 async def serve_backgrounds(request: Request):
     """Sandbox page for prototyping background effects. No auth required."""
-    return _serve_html_with_nonce(request, abs_join(BASE_DIR, "static/backgrounds.html"))
+    return serve_html_with_nonce(request, abs_join(BASE_DIR, "static/backgrounds.html"))
 
 @app.get("/login")
 async def serve_login(request: Request):
     if not AUTH_ENABLED:
         return RedirectResponse(url="/", status_code=302)
-    return _serve_html_with_nonce(request, abs_join(BASE_DIR, "static/login.html"))
+    return serve_html_with_nonce(request, abs_join(BASE_DIR, "static/login.html"))
 
 @app.get("/api/version")
 async def get_version():
@@ -1001,17 +1025,21 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
 
-    # Keep-alive: ping endpoints every 60 seconds to prevent cold starts
-    async def _keepalive_loop():
-        while True:
-            try:
-                await asyncio.sleep(60)
-                await _warmup_endpoints()
-            except Exception as e:
-                logger.warning(f"Keepalive loop error: {e}")
-                await asyncio.sleep(300)  # Back off on error
+    # Keep-alive is opt-in. The ping path performs model discovery, and when
+    # stale LAN endpoints are configured it can add periodic backend pressure
+    # that delays unrelated UI requests such as Notes/Documents.
+    _keepalive_enabled = str(os.getenv("ODYSSEUS_MODEL_KEEPALIVE", "")).lower() in {"1", "true", "yes", "on"}
+    if _keepalive_enabled:
+        async def _keepalive_loop():
+            while True:
+                try:
+                    await asyncio.sleep(60)
+                    await _warmup_endpoints()
+                except Exception as e:
+                    logger.warning(f"Keepalive loop error: {e}")
+                    await asyncio.sleep(300)  # Back off on error
 
-    _startup_tasks.append(asyncio.create_task(_keepalive_loop()))
+        _startup_tasks.append(asyncio.create_task(_keepalive_loop()))
 
     async def _ensure_default_tasks():
         # Create/reconcile default automation tasks + personal assistant for every user.

core/atomic_io.py

@@ -34,6 +34,8 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) ->
 
 
 def atomic_write_text(path: str, text: str) -> None:
+    if not isinstance(text, str):
+        raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
     tmp = f"{path}.tmp.{os.getpid()}"
     with open(tmp, "w", encoding="utf-8") as f:

core/auth.py

@@ -176,16 +176,17 @@ class AuthManager:
                 )
                 old_user = "admin"
             old_hash = self._config["password_hash"]
-            self._config = {
-                "users": {
-                    old_user: {
-                        "password_hash": old_hash,
-                        "created": time.time(),
-                        "is_admin": True,
+            with self._config_lock:
+                self._config = {
+                    "users": {
+                        old_user: {
+                            "password_hash": old_hash,
+                            "created": time.time(),
+                            "is_admin": True,
+                        }
                     }
                 }
-            }
-            self._save()
+                self._save()
             logger.info(f"Migrated single-user auth to multi-user (admin: {old_user})")
 
     def _drop_reserved_loaded_users(self):
@@ -204,8 +205,9 @@ class AuthManager:
                 continue
             normalized[key] = data
         if removed or normalized != users:
-            self._config["users"] = normalized
-            self._save()
+            with self._config_lock:
+                self._config["users"] = normalized
+                self._save()
         if removed:
             logger.warning(
                 "Removed reserved username(s) from auth config: %s",

core/database.py

@@ -276,6 +276,7 @@ class GalleryImage(TimestampMixin, Base):
     id         = Column(String, primary_key=True, index=True)
     filename   = Column(String, nullable=False, unique=True)
     prompt     = Column(Text, nullable=False, default="")
+    caption    = Column(Text, nullable=True, default="")
     model      = Column(String, nullable=True)
     size       = Column(String, nullable=True)
     quality    = Column(String, nullable=True)
@@ -1182,6 +1183,29 @@ def _migrate_add_multiuser_owner_columns():
     _migrate_add_owner_to_table("documents", "ix_documents_owner")
 
 
+def _migrate_add_gallery_caption_column():
+    """Add OCR/vision caption storage for gallery images."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(gallery_images)").fetchall()]
+        if columns and "caption" not in columns:
+            conn.execute("ALTER TABLE gallery_images ADD COLUMN caption TEXT DEFAULT ''")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added caption column to gallery_images")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration gallery caption column failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+
 def _migrate_add_api_token_scopes_column():
     """Add API token scopes for existing installs.
 
@@ -1670,6 +1694,7 @@ class CalendarEvent(TimestampMixin, Base):
     # `Z`-suffix on serialization so the frontend interprets correctly.
     is_utc      = Column(Boolean, default=False, nullable=False)
     rrule       = Column(String, default="")
+    recurrence_exdates = Column(Text, default="")  # JSON list of skipped occurrence starts
     color       = Column(String, nullable=True)  # per-event color override
     status      = Column(String, default="confirmed")  # confirmed, cancelled
     importance  = Column(String, default="normal")    # low | normal | high | critical
@@ -1811,6 +1836,7 @@ def init_db():
     _migrate_add_token_columns()
     _migrate_add_mode_column()
     _migrate_add_multiuser_owner_columns()
+    _migrate_add_gallery_caption_column()
     _migrate_add_api_token_scopes_column()
     _migrate_backfill_document_owner_from_session()
     _migrate_assign_legacy_owner()
@@ -1833,6 +1859,7 @@ def init_db():
     _migrate_add_calendar_origin()
     _migrate_add_calendar_account_id()
     _migrate_add_caldav_sync_columns()
+    _migrate_add_calendar_recurrence_exdates()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2184,6 +2211,28 @@ def _migrate_add_calendar_metadata():
         except Exception:
             pass
 
+
+def _migrate_add_calendar_recurrence_exdates():
+    """Add skipped recurrence occurrences for deleting one instance of a series."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
+        if columns and "recurrence_exdates" not in columns:
+            conn.execute("ALTER TABLE calendar_events ADD COLUMN recurrence_exdates TEXT DEFAULT ''")
+        conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"calendar_events recurrence_exdates migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def get_db():
     """
     Dependency to get a database session.

core/exceptions.py

@@ -1,4 +1,4 @@
-# src/exceptions.py
+# core/exceptions.py
 """Custom exceptions for the application."""
 
 class SessionNotFoundError(Exception):

core/log_safety.py

@@ -0,0 +1,27 @@
+"""Helpers for keeping sensitive data out of logs.
+
+Endpoint URLs configured by admins can embed credentials in the userinfo
+(``https://user:pass@host``) or query string (``?api_key=...``). Logging them
+raw leaks those secrets, so route/diagnostic logs run URLs through
+``redact_url`` first. Reconstructing the URL without userinfo/query/fragment
+also doubles as a sanitizer barrier for CodeQL's clear-text-logging query.
+"""
+
+from urllib.parse import urlparse, urlunparse
+
+
+def redact_url(url: str) -> str:
+    """Return a URL safe for logs by removing userinfo and query/fragment.
+
+    Keeps scheme, host, port and path so logs stay useful for debugging.
+    """
+    try:
+        parsed = urlparse(url or "")
+        host = parsed.hostname or ""
+        if ":" in host:  # IPv6 literal — re-bracket so host:port stays unambiguous
+            host = f"[{host}]"
+        if parsed.port:
+            host = f"{host}:{parsed.port}"
+        return urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
+    except Exception:
+        return "<endpoint>"

core/middleware.py

@@ -117,7 +117,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
                 f"script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net; "
                 "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
                 "font-src 'self' https://cdn.jsdelivr.net; "
-                "img-src 'self' data: blob:; "
+                "img-src 'self' data: blob: https:; "
                 "media-src 'self' blob:; "
                 "connect-src 'self'; "
                 "frame-src 'self'; "

core/session_manager.py

@@ -40,7 +40,18 @@ def _parse_msg_content(raw):
     if isinstance(raw, str) and raw.startswith('[{') and '"type"' in raw:
         try:
             parsed = json.loads(raw)
-            if isinstance(parsed, list) and all(isinstance(p, dict) for p in parsed):
+            # Only treat as serialized multimodal content when EVERY element is
+            # a dict whose "type" is a recognized content-block kind. Otherwise a
+            # plain text message that merely *looks* like a JSON array of objects
+            # (e.g. a user pasting an API schema/sample with a "type" field) was
+            # silently parsed back into a list, destroying the original string.
+            _BLOCK_TYPES = {
+                "text", "image", "image_url", "audio", "input_audio",
+                "input_image", "document", "file",
+            }
+            if (isinstance(parsed, list) and parsed
+                    and all(isinstance(p, dict) and p.get("type") in _BLOCK_TYPES
+                            for p in parsed)):
                 return parsed
         except (json.JSONDecodeError, ValueError):
             pass

docker/build-realesrgan-wheels.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Build patched wheels for Real-ESRGAN's unmaintained dependencies.
+#
+# basicsr / gfpgan / facexlib (xinntao, last released 2022) read their version
+# in setup.py with:
+#
+#     exec(compile(f.read(), version_file, 'exec'))
+#     return locals()['__version__']
+#
+# Python 3.13+ implements PEP 667: locals() inside a function returns an
+# independent snapshot that exec() can no longer mutate, so the read raises
+# `KeyError: '__version__'` and the sdist build fails. That is why the Cookbook
+# "install realesrgan" button dies on the python:3.14 image. The packages have
+# no fixed release, so we patch get_version() to exec into an explicit namespace
+# dict (works on every Python) and build wheels from the patched source.
+#
+# Usage: build-realesrgan-wheels.sh [OUTPUT_DIR]   (default: /wheels)
+set -euo pipefail
+
+OUT="${1:-/wheels}"
+mkdir -p "$OUT"
+
+work="$(mktemp -d)"
+trap 'rm -rf "$work"' EXIT
+cd "$work"
+
+# Pinned to the versions Real-ESRGAN 0.3.0 resolves to.
+SPECS="basicsr==1.4.2 gfpgan==1.3.8 facexlib==0.3.0"
+
+for spec in $SPECS; do
+  name="${spec%%==*}"
+  ver="${spec##*==}"
+  # pip download builds metadata (and trips the same bug), so fetch the raw
+  # sdist URL from the PyPI JSON API instead.
+  url="$(python - "$name" "$ver" <<'PY'
+import json, sys, urllib.request
+name, ver = sys.argv[1], sys.argv[2]
+data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{ver}/json"))
+for f in data["urls"]:
+    if f["packagetype"] == "sdist":
+        print(f["url"]); break
+else:
+    sys.exit(f"no sdist found for {name}=={ver}")
+PY
+)"
+  echo ">> fetching ${name} ${ver}: ${url}"
+  curl -fsSL "$url" -o "${name}.tar.gz"
+  tar xzf "${name}.tar.gz"
+done
+
+echo ">> patching get_version()"
+python - <<'PY'
+import pathlib
+old_exec = "exec(compile(f.read(), version_file, 'exec'))"
+new_exec = "_ver_ns = {}\n        exec(compile(f.read(), version_file, 'exec'), _ver_ns)"
+old_ret = "return locals()['__version__']"
+new_ret = "return _ver_ns['__version__']"
+patched = 0
+for setup in pathlib.Path(".").glob("*/setup.py"):
+    s = setup.read_text()
+    if old_exec in s and old_ret in s:
+        setup.write_text(s.replace(old_exec, new_exec).replace(old_ret, new_ret))
+        print("   patched", setup)
+        patched += 1
+assert patched == 3, f"expected to patch 3 setup.py files, patched {patched}"
+PY
+
+echo ">> building wheels into ${OUT}"
+pip wheel --no-deps -w "$OUT" ./basicsr-* ./gfpgan-* ./facexlib-*
+ls -l "$OUT"

docker/entrypoint.sh

@@ -26,6 +26,27 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then
     useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus
 fi
 
+ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)"
+[ -z "$ODY_USER" ] && ODY_USER=odysseus
+
+# Docker-socket group plumbing for the explicit host-Docker overlay. When
+# opted in, the socket is owned by root:<host docker gid>. Add the app user
+# to that group and later call gosu by username so supplementary groups are
+# retained.
+DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}"
+if [ "${ODYSSEUS_ENABLE_HOST_DOCKER:-}" = "true" ] && [ -S "$DOCKER_SOCK" ]; then
+    SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')"
+    if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then
+        if ! getent group "$SOCK_GID" >/dev/null 2>&1; then
+            groupadd -g "$SOCK_GID" docker_host || true
+        fi
+        SOCK_GROUP="$(getent group "$SOCK_GID" | cut -d: -f1)"
+        if [ -n "$SOCK_GROUP" ]; then
+            usermod -aG "$SOCK_GROUP" "$ODY_USER" 2>/dev/null || true
+        fi
+    fi
+fi
+
 mount_root_for() {
     awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true
 }
@@ -103,6 +124,7 @@ for cu in \
         break
     fi
 done
+
 # Disable the FlashInfer JIT sampler unconditionally — it is sampler-only
 # and has no impact on the attention path, but requires nvcc + matching
 # CUDA headers at startup. Without this, vLLM crashes with "Could not find
@@ -116,9 +138,9 @@ export PATH="/app/.local/bin:$PATH"
 # Run first-time setup as the app user so data/ files get the right ownership.
 # setup.py is idempotent — skips auth.json / .env if they already exist.
 # || true so a setup failure never prevents the container from starting.
-"$GOSU_BIN" "$PUID:$PGID" "$PYTHON_BIN" /app/setup.py || true
+"$GOSU_BIN" "$ODY_USER" "$PYTHON_BIN" /app/setup.py || true
 
 # Drop root and run the actual app. `gosu` is preferred over `su` /
 # `sudo` because it cleans up the process tree (no extra shell layer)
 # so signals (SIGTERM from `docker stop`) reach uvicorn directly.
-exec "$GOSU_BIN" "$PUID:$PGID" "$@"
+exec "$GOSU_BIN" "$ODY_USER" "$@"

docker/host-docker.yml

@@ -0,0 +1,12 @@
+# High-trust host Docker access. Enable only when local Docker-daemon
+# management from Cookbook is required and you accept that raw socket access
+# grants broad control over the host Docker daemon.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
+#   DOCKER_GID=<numeric host Docker group id>
+services:
+  odysseus:
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    group_add: ["${DOCKER_GID:-963}"]
+    environment:
+      - ODYSSEUS_ENABLE_HOST_DOCKER=true

docs/setup.md

@@ -15,7 +15,7 @@ On first setup, Odysseus creates an admin account (`admin` unless
 For Docker installs, the same line is in `docker compose logs odysseus`.
 Use that for the first login, then change it in **Settings**.
 
-Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and
+Contributing? See [CONTRIBUTING.md](../CONTRIBUTING.md) for setup, testing, and
 pull request guidelines.
 
 ### Docker (recommended)
@@ -99,6 +99,33 @@ Odysseus SSH key and add the public key to the remote server's
 ssh-copy-id -i data/ssh/id_ed25519.pub user@server
 ```
 
+**Host Docker access (explicit opt-in).** Default Docker Compose intentionally
+does not mount `/var/run/docker.sock`. You can still connect Odysseus to
+existing Ollama, vLLM, and other OpenAI-compatible endpoints without Docker
+socket access.
+
+Cookbook/local Docker-daemon management requires the opt-in overlay below. Raw
+Docker socket access is high-trust because it can effectively grant broad
+control over the host Docker daemon. Remote server Docker workflows over SSH
+remain preferred.
+
+Place these values in `.env`, or export them in the shell before running
+`docker compose`:
+
+```bash
+COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
+DOCKER_GID=<host docker group gid>
+```
+
+Combine host Docker access with a GPU overlay when both are intentionally
+required:
+
+```bash
+COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
+# or
+COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
+```
+
 **Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
 only detect GPUs that Docker exposes to the container — if the host runtime or
 device passthrough is not configured, Cookbook sees the iGPU, another card, or
@@ -250,6 +277,19 @@ python -m uvicorn app:app --host 127.0.0.1 --port 7000
 If `python` points at an older interpreter, use `py -3.12` (or another installed
 3.11+ version) for the venv step.
 
+**Exposing on a LAN/Tailscale (Windows):** the launcher binds to `127.0.0.1` and
+does **not** read `APP_BIND` / `ODYSSEUS_HOST` from `.env`, so editing `.env`
+alone leaves the native Windows server on loopback. Pass the launcher's
+`-BindHost` flag instead:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 -BindHost 0.0.0.0
+```
+
+The manual `uvicorn` command takes the same address as `--host 0.0.0.0`. Bind
+outside loopback only for a trusted LAN/VPN such as Tailscale: keep
+`AUTH_ENABLED=true` and do not expose the port directly to the public internet.
+
 **Requirements:** Python 3.11+. The core app (chat, agent, memory, documents,
 email, calendar, deep research) runs fully native. For full **Cookbook** background
 model downloads and the agent shell tool, also install
@@ -286,6 +326,16 @@ To expose Odysseus on a local network or Tailscale with HTTPS:
    ```
 4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings).
 
+### Common self-host traps (30-second fixes)
+A grab-bag of small gotchas that otherwise turn into long debugging sessions.
+
+- **`AUTH_ENABLED=false` is ignored / you're still forced to log in (Windows).** If you edited `.env` in Notepad it may have saved a UTF-8 **BOM**, turning the first key into `AUTH_ENABLED` so it is never matched. Odysseus loads `.env` with `encoding="utf-8-sig"` to tolerate a leading BOM, but the safe fix is to re-save `.env` as **UTF-8 without BOM** (VS Code: *Save with Encoding → UTF-8*).
+- **macOS: the app isn't at `http://localhost:7000`.** macOS AirPlay Receiver usually holds port `7000`, so the macOS start script serves on **`7860`** instead — open `http://localhost:7860`. To use `7000`, free it (System Settings → General → AirDrop & Handoff → turn off *AirPlay Receiver*) and set `APP_PORT=7000`.
+- **Copy buttons do nothing over a plain-HTTP Tailscale/LAN URL.** Browsers only expose the clipboard API (`navigator.clipboard`) on **secure origins** — HTTPS, or `localhost`. Over `http://100.x.y.z:7860` it is blocked. Serve over HTTPS (see *HTTPS + LAN/Tailscale exposure* above); `localhost` is exempt, so copy still works on the host itself.
+- **Self-hosted ntfy reminders don't reach your phone.** Two things: (1) the bundled ntfy binds to loopback by default — to reach it from your phone set `NTFY_BIND` to your host/Tailscale IP and `NTFY_BASE_URL` to the same server URL in `.env`, then recreate the ntfy container (see the `NTFY_*` block in `.env.example`); (2) in the ntfy **Android** app, subscribe to the topic with **Instant delivery** enabled — non-`ntfy.sh` servers don't get instant push otherwise.
+- **Local mail (Dovecot) login fails: "Plaintext authentication disallowed on non-encrypted connections."** Your IMAP/SMTP server is refusing cleartext auth over an unencrypted link. Prefer enabling TLS on the mail server; on a trusted LAN only, you can allow cleartext (Dovecot: `disable_plaintext_auth = no`).
+- **Calendar/contacts (Radicale) won't sync.** Point Odysseus at the **full collection URL** with its trailing slash — e.g. `http://host:5232/<user>/<collection-id>/` — not just the server root. Radicale shows this address for each calendar/address book in its web UI.
+
 ### Optional Dependencies
 `requirements-optional.txt` contains packages that unlock extra features. It is not installed by default.
 

licenses/OpenDyslexic-OFL.txt

@@ -0,0 +1,94 @@
+Copyright (c) 2019-07-29, Abbie Gonzalez (https://abbiecod.es|support@abbiecod.es),
+with Reserved Font Name OpenDyslexic.
+Copyright (c) 12/2012 - 2019
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.

mcp_servers/email_server.py

@@ -538,6 +538,148 @@ def _get_cached_summaries():
         return {}
 
 
+def _fixture_email_file() -> Path:
+    return DATA_DIR / "fixture_email_messages.json"
+
+
+def _fixture_email_enabled() -> bool:
+    return _fixture_email_file().exists()
+
+
+def _parse_fixture_date(raw_date: str) -> tuple[str, float]:
+    if not raw_date:
+        return "", 0.0
+    parsed = None
+    try:
+        parsed = datetime.fromisoformat(str(raw_date).replace("Z", "+00:00"))
+    except Exception:
+        try:
+            parsed = email.utils.parsedate_to_datetime(str(raw_date))
+        except Exception:
+            parsed = None
+    if parsed:
+        return parsed.isoformat(), parsed.timestamp()
+    return str(raw_date), 0.0
+
+
+def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict:
+    sender = str(row.get("from") or "Fixture Sender <fixture@example.invalid>")
+    sender_name, sender_addr = email.utils.parseaddr(sender)
+    date_str, date_epoch = _parse_fixture_date(str(row.get("date") or ""))
+    subject = str(row.get("subject") or "(no subject)")
+    body = str(row.get("body") or "")
+    owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default")
+    uid = str(uid_num)
+    return {
+        "uid": uid,
+        "message_id": f"<fixture-email-{uid}-{owner_key}@fixtures.odysseus.local>",
+        "subject": subject,
+        "from": sender_name or sender_addr or sender,
+        "from_address": sender_addr,
+        "date": date_str,
+        "date_epoch": date_epoch,
+        "summary": body[:240],
+        "body": body,
+        "account": "Fixture Inbox",
+        "account_email": owner or str(row.get("owner") or ""),
+        "account_id": "fixture-email",
+        "attachments": [],
+    }
+
+
+def _fixture_email_rows(owner: str | None = None) -> list[dict]:
+    path = _fixture_email_file()
+    if not path.exists():
+        return []
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return []
+    rows = raw.get("messages") if isinstance(raw, dict) else raw
+    out = []
+    owner = str(owner or "").strip()
+    for i, row in enumerate(rows if isinstance(rows, list) else [], start=1):
+        if not isinstance(row, dict):
+            continue
+        row_owner = str(row.get("owner") or "").strip()
+        if owner and row_owner and row_owner != owner:
+            continue
+        out.append(_fixture_email_record(row, i, owner or row_owner))
+    out.sort(key=lambda item: item.get("date_epoch") or 0, reverse=True)
+    return out
+
+
+def _fixture_account_rows() -> list[dict]:
+    if not _fixture_email_enabled():
+        return []
+    owner = _current_owner()
+    owners = []
+    for row in _fixture_email_rows(owner or None):
+        email_addr = row.get("account_email") or owner or "fixture@fixtures.odysseus.local"
+        if email_addr not in owners:
+            owners.append(email_addr)
+    if not owners:
+        owners = [owner or "fixture@fixtures.odysseus.local"]
+    return [
+        {
+            "id": "fixture-email",
+            "owner": owner or owners[0],
+            "name": "Fixture Inbox",
+            "is_default": True,
+            "imap_user": owners[0],
+            "from_address": owners[0],
+        }
+    ]
+
+
+def _fixture_email_matches(item: dict, query: str) -> bool:
+    if not query:
+        return True
+    terms = [term for term in re.split(r"\W+", str(query).lower()) if term]
+    haystack = "\n".join(
+        str(item.get(key) or "")
+        for key in ("subject", "from", "from_address", "body", "summary")
+    ).lower()
+    return all(term in haystack for term in terms)
+
+
+def _fixture_list_emails(folder="INBOX", max_results=20, unresponded_only=False,
+                         unread_only=False, account=None) -> list[dict] | None:
+    if not _fixture_email_enabled():
+        return None
+    if account and str(account).strip().lower() not in {
+        "fixture-email",
+        "fixture inbox",
+        "fixture",
+        str(_current_owner()).lower(),
+    }:
+        return []
+    if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
+        return []
+    return _fixture_email_rows(_current_owner())[: int(max_results or 20)]
+
+
+def _fixture_search_emails(query, folders=None, max_results=20, account=None) -> list[dict] | None:
+    if not _fixture_email_enabled():
+        return None
+    rows = _fixture_list_emails("INBOX", max_results=1000, account=account) or []
+    out = [dict(row, _folder="INBOX") for row in rows if _fixture_email_matches(row, str(query or ""))]
+    return out[: int(max_results or 20)]
+
+
+def _fixture_read_email(uid=None, message_id=None, folder="INBOX", account=None) -> dict | None:
+    if not _fixture_email_enabled():
+        return None
+    if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
+        return {"error": f"Email UID {uid or message_id} not found"}
+    for item in _fixture_email_rows(_current_owner()):
+        if uid and str(item.get("uid")) == str(uid):
+            return item
+        if message_id and str(item.get("message_id")) == str(message_id):
+            return item
+    return {"error": f"Email not found with UID/Message-ID: {uid or message_id}"}
+
+
 # ── Tool implementations ──
 
 
@@ -548,6 +690,9 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False,
     Pass unread_only=True and/or unresponded_only=True for attention scans.
     account selects mailbox (None = default).
     """
+    fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, account)
+    if fixture is not None:
+        return fixture
     conn = None
     try:
         conn = _imap_connect(account)
@@ -629,6 +774,9 @@ def _result_sort_time(result: dict) -> datetime:
 
 def _list_emails_across_accounts(folder="INBOX", max_results=20,
                                  unresponded_only=False, unread_only=False):
+    fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, None)
+    if fixture is not None:
+        return fixture, []
     rows = _list_accounts_raw()
     combined = []
     errors = []
@@ -662,6 +810,9 @@ def _search_emails(query, folders=None, max_results=20, account=None):
     _list_emails plus an `_folder` tag."""
     if not query or not str(query).strip():
         return []
+    fixture = _fixture_search_emails(query, folders=folders, max_results=max_results, account=account)
+    if fixture is not None:
+        return fixture
     q = str(query).replace("\\", "\\\\").replace('"', '\\"')
     # Mail clients commonly use OR FROM/SUBJECT/TEXT to match either field.
     # IMAP SEARCH OR is binary, so we nest it.
@@ -784,6 +935,9 @@ def _extract_attachment_to_disk(msg, index, target_dir):
 
 def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
     """Read full email content by UID or message-ID. account = mailbox selector."""
+    fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=account)
+    if fixture is not None:
+        return fixture
     cfg = _load_config(account)
     conn = None
     try:
@@ -837,6 +991,9 @@ def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
 
 
 def _read_email_across_accounts(uid=None, message_id=None, folder="INBOX"):
+    fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=None)
+    if fixture is not None:
+        return fixture
     rows = _list_accounts_raw()
     matches = []
     errors = []
@@ -1775,9 +1932,10 @@ async def list_tools() -> list[Tool]:
         Tool(
             name="reply_to_email",
             description=(
-                "Reply to an existing email by UID. This sends immediately; for normal "
-                "assistant-written replies, prefer draft_email_reply so the user can "
-                "review and send from Odysseus. Automatically threads the reply with "
+                "Reply to an existing email by UID. This sends immediately. Do NOT use "
+                "for normal 'write/draft a reply saying X' requests; use "
+                "draft_email_reply so the user can review and send from Odysseus. "
+                "Only use this when the user explicitly says to send now. Automatically threads the reply with "
                 "In-Reply-To and References headers, prefixes 'Re:' on the subject, and "
                 "uses the original sender as the recipient. Set reply_all=true to also CC "
                 "the original To/Cc recipients. For follow-up 'reply ...' requests, use "
@@ -1991,6 +2149,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
 
         if name == "list_email_accounts":
             rows = _filter_accounts_for_owner(all_db_accounts)
+            if not rows:
+                rows = _fixture_account_rows()
             if not rows:
                 if all_db_accounts and owner:
                     return [TextContent(type="text", text="No email accounts configured for this owner.")]

mcp_servers/image_gen_server.py

@@ -73,7 +73,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
         if not model_spec:
             for candidate in ("gpt-image-1.5", "gpt-image-1", "dall-e-3"):
                 try:
-                    _resolve_model(candidate)
+                    await asyncio.to_thread(_resolve_model, candidate)
                     model_spec = candidate
                     break
                 except ValueError:
@@ -81,7 +81,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
             if not model_spec:
                 return [TextContent(type="text", text="Error: No image model found. Configure one in Admin.")]
 
-        url, model_id, headers = _resolve_model(model_spec)
+        url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec)
 
         is_gpt_image = "gpt-image" in model_id.lower()
         base_url = url.replace("/chat/completions", "").replace("/v1/messages", "").rstrip("/")

requirements.txt

@@ -3,8 +3,9 @@ uvicorn
 python-multipart
 python-dotenv
 httpx
+httpcore>=1.0.9,<2.0
 pydantic>=2.13.4
-pydantic-settings>=2.14.1
+pydantic-settings>=2.14.2
 SQLAlchemy
 pypdf
 beautifulsoup4

routes/calendar_routes.py

@@ -1,6 +1,7 @@
 """Calendar routes — local SQLite-backed calendar CRUD."""
 
 import logging
+import json
 import re
 import uuid
 from datetime import datetime, date, timedelta
@@ -34,6 +35,24 @@ def _ics_naive_dtstart(dt):
         return datetime(dt.year, dt.month, dt.day)
     return dt
 
+
+def _ensure_positive_duration(start_dt, end_dt, all_day):
+    """Clamp an imported event's end so it has a positive duration.
+
+    Some .ics exporters write a single-day all-day event with DTEND equal to
+    DTSTART (treating DTEND as inclusive rather than the RFC 5545 exclusive
+    bound). Stored verbatim that produces a zero-duration row, which the
+    list_events overlap filter (dtstart < end AND dtend > start) silently
+    drops — the event never appears on the calendar even though the web UI
+    would otherwise show it. Normalize a non-positive end to the same default
+    span used when DTEND is absent: one day for all-day events, one hour
+    otherwise.
+    """
+    if end_dt <= start_dt:
+        return start_dt + (timedelta(days=1) if all_day else timedelta(hours=1))
+    return end_dt
+
+
 # Single-user fallback identity. Used only when:
 #   1. The app is configured for single-user (no auth middleware), AND
 #   2. The request didn't resolve to an authenticated user.
@@ -434,6 +453,20 @@ def _parse_dt(s: str) -> datetime:
         if t is not None:
             return base.replace(hour=t[0], minute=t[1])
 
+    # time-first: "3pm today", "9am tomorrow", "11pm tonight"
+    # (parity with parse_due_for_user, which handles these via the same form)
+    m = _re.match(r'^(.+?)\s+(today|tonight|tomorrow|tmrw|yesterday)$', lower)
+    if m:
+        time_part, word = m.group(1).strip(), m.group(2)
+        base = today
+        if word in ("tomorrow", "tmrw"):
+            base = today + timedelta(days=1)
+        elif word == "yesterday":
+            base = today - timedelta(days=1)
+        t = _parse_time(time_part)
+        if t is not None:
+            return base.replace(hour=t[0], minute=t[1])
+
     # next <weekday> [at] TIME
     weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
     m = _re.match(r'^next\s+(\w+)(?:\s+at)?\s*(.*)$', lower)
@@ -509,6 +542,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "description": ev.description or "",
         "location": ev.location or "",
         "rrule": ev.rrule or "",
+        "recurrence_exdates": _recurrence_exdates(ev),
         "calendar": ev.calendar.name if ev.calendar else "",
         "calendar_href": ev.calendar_id,
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
@@ -522,6 +556,28 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
 _RRULE_EXPANSION_LIMIT = 1000
 
 
+def _recurrence_exdates(ev: CalendarEvent) -> list[str]:
+    raw = getattr(ev, "recurrence_exdates", "") or ""
+    if not raw:
+        return []
+    try:
+        values = json.loads(raw)
+    except Exception:
+        return []
+    if not isinstance(values, list):
+        return []
+    return [str(v) for v in values if isinstance(v, str) and v.strip()]
+
+
+def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
+    if "::" not in uid:
+        return ""
+    suffix = uid.split("::", 1)[1]
+    if ev.all_day:
+        return suffix[:10]
+    return suffix[:16]
+
+
 def _expand_rrule(
     ev: CalendarEvent, start: datetime, end: datetime
 ) -> List[dict]:
@@ -586,6 +642,7 @@ def _expand_rrule(
     results = []
     truncated = False
     base = _event_to_dict(ev)
+    exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
         if occ_start >= end:
@@ -606,8 +663,13 @@ def _expand_rrule(
         # Build the compound uid: {base_uid}::{date} or ::{datetime}
         if ev.all_day:
             occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}"
+            exdate_key = occ_start.strftime("%Y-%m-%d")
         else:
             occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%dT%H:%M')}"
+            exdate_key = occ_start.strftime("%Y-%m-%dT%H:%M")
+
+        if exdate_key in exdates:
+            continue
 
         d = dict(base)
         d["uid"] = occ_uid
@@ -1118,7 +1180,7 @@ def setup_calendar_routes() -> APIRouter:
             db.close()
 
     @router.delete("/events/{uid}")
-    async def delete_event(request: Request, uid: str):
+    async def delete_event(request: Request, uid: str, scope: str = "series"):
         owner = _require_user(request)
         try:
             base_uid = _resolve_base_uid(uid)
@@ -1127,7 +1189,22 @@ def setup_calendar_routes() -> APIRouter:
         db = SessionLocal()
         try:
             ev = _get_or_404_event(db, base_uid, owner)
+            is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if is_occurrence_delete:
+                key = _occurrence_exdate_key(uid, ev)
+                if not key:
+                    raise HTTPException(400, "Invalid recurring occurrence uid")
+                exdates = _recurrence_exdates(ev)
+                if key not in exdates:
+                    exdates.append(key)
+                ev.recurrence_exdates = json.dumps(sorted(exdates))
+                if is_caldav:
+                    ev.caldav_sync_pending = "update"
+                db.commit()
+                if is_caldav:
+                    await _push_caldav_event_after_commit(owner, base_uid, "update")
+                return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
             db.delete(ev)
@@ -1226,7 +1303,7 @@ def setup_calendar_routes() -> APIRouter:
                 db.commit()
                 db.refresh(target_cal)
 
-            imported = skipped = 0
+            imported = skipped = repaired = 0
             for comp in cal_data.walk():
                 if comp.name != "VEVENT":
                     continue
@@ -1262,6 +1339,18 @@ def setup_calendar_routes() -> APIRouter:
                         .first()
                     )
                     if existing:
+                        # An import predating the clamp below may have stored
+                        # this same event with a non-positive duration, which
+                        # the list_events overlap filter hides. Re-importing
+                        # lands here and would skip without touching that row,
+                        # so the event would stay invisible. Backfill the clamp
+                        # onto the stored row before skipping it.
+                        fixed_end = _ensure_positive_duration(
+                            existing.dtstart, existing.dtend, bool(existing.all_day)
+                        )
+                        if fixed_end != existing.dtend:
+                            existing.dtend = fixed_end
+                            repaired += 1
                         skipped += 1
                         continue
 
@@ -1295,6 +1384,8 @@ def setup_calendar_routes() -> APIRouter:
                     else:
                         end_dt = start_dt + timedelta(hours=1)
 
+                end_dt = _ensure_positive_duration(start_dt, end_dt, all_day)
+
                 ev = CalendarEvent(
                     uid=uid_val,
                     calendar_id=target_cal.id,
@@ -1315,6 +1406,7 @@ def setup_calendar_routes() -> APIRouter:
                 "ok": True,
                 "imported": imported,
                 "skipped": skipped,
+                "repaired": repaired,
                 "calendar": cal_display,
                 "calendar_id": target_cal.id,
             }

routes/chat_helpers.py

@@ -22,6 +22,31 @@ from fastapi import HTTPException
 
 logger = logging.getLogger(__name__)
 
+_CASUAL_OPENING_RE = re.compile(
+    r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
+    r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
+    re.IGNORECASE,
+)
+_CASUAL_BLOCKLIST_RE = re.compile(
+    r"\b(?:cookbook|serve|serving|launch|start|vllm|sglang|llama\.?cpp|ollama|"
+    r"download|model|email|document|doc|note|calendar|task|search|web|research|"
+    r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
+    re.IGNORECASE,
+)
+
+
+def _is_casual_low_signal(text: str) -> bool:
+    """Short greetings/slang should not pull memory, skills, RAG, or docs."""
+    s = str(text or "").strip()
+    m = _CASUAL_OPENING_RE.match(s)
+    if not m:
+        return False
+    tail = m.group("tail") or ""
+    if _CASUAL_BLOCKLIST_RE.search(tail):
+        return False
+    tail_words = re.findall(r"[A-Za-z0-9_'-]+", tail)
+    return len(tail_words) <= 2
+
 
 # Strong references to in-flight fire-and-forget tasks scheduled from this
 # module. asyncio only keeps weak references to tasks created via
@@ -79,6 +104,9 @@ class ChatContext:
     # The chat route emits a doc_update SSE event for each before streaming
     # begins, so the editor pane switches to the new doc immediately.
     auto_opened_docs: list = field(default_factory=list)
+    # Uploads attached to this user turn, resolved and owner-checked for the
+    # agent's private context. This is not emitted to the browser.
+    uploaded_files: list = field(default_factory=list)
 
 
 # ── Helpers ────────────────────────────────────────────────────────────── #
@@ -341,6 +369,59 @@ async def preprocess(
     )
 
 
+def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[str]) -> list[dict]:
+    """Resolve current-turn upload IDs into a small tool-facing manifest.
+
+    The chat UI already sends attachment ids, and preprocessing inlines as much
+    text as fits. Agent mode still needs a discoverable bridge for files whose
+    content was truncated/omitted or when the model chooses file tools. Only
+    owner-authorized uploads are included, and paths must remain inside the
+    configured upload directory.
+    """
+    if not att_ids or not upload_handler or not hasattr(upload_handler, "resolve_upload"):
+        return []
+
+    def _read_file_can_open(path: str) -> bool:
+        try:
+            from src.tool_execution import _resolve_tool_path
+
+            return _resolve_tool_path(path) == os.path.realpath(path)
+        except Exception:
+            return False
+
+    manifest: list[dict] = []
+    for att_id in att_ids:
+        try:
+            info = upload_handler.resolve_upload(str(att_id), owner=owner)
+        except Exception:
+            logger.debug("Failed to resolve upload %r for agent manifest", att_id, exc_info=True)
+            continue
+        if not isinstance(info, dict):
+            continue
+
+        path = info.get("path")
+        if path:
+            try:
+                inside = True
+                if hasattr(upload_handler, "_inside_upload_dir"):
+                    inside = bool(upload_handler._inside_upload_dir(path))
+                elif hasattr(upload_handler, "inside_base_dir"):
+                    inside = bool(upload_handler.inside_base_dir(path))
+                if not inside or not os.path.exists(path) or not _read_file_can_open(path):
+                    path = None
+            except Exception:
+                path = None
+
+        manifest.append({
+            "id": info.get("id") or str(att_id),
+            "name": info.get("name") or info.get("original_name") or str(att_id),
+            "mime": info.get("mime", ""),
+            "size": info.get("size", 0),
+            "path": path,
+        })
+    return manifest
+
+
 def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
     """Add user message to session history and update session name.
     In incognito mode, still add to in-memory history (for conversation context)
@@ -588,6 +669,12 @@ async def build_chat_context(
     # bearer-token chat requests use the token owner instead of the "api" sentinel.
     user = effective_user(request)
     uprefs = load_prefs_for_user(user)
+    uploaded_files = build_uploaded_file_manifest(
+        att_ids or [],
+        getattr(chat_handler, "upload_handler", None),
+        getattr(sess, "owner", None),
+    )
+    casual_low_signal = _is_casual_low_signal(message)
 
     # Memory enabled?
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
@@ -597,6 +684,9 @@ async def build_chat_context(
     if not allow_tool_preprocessing:
         mem_enabled = False
         skills_enabled = False
+    if casual_low_signal:
+        mem_enabled = False
+        skills_enabled = False
     logger.debug(
         "Memory enabled=%s for user=%s (incognito=%s, no_memory=%s, pref=%s)",
         mem_enabled, user, incognito, no_memory, uprefs.get("memory_enabled", "NOT_SET"),
@@ -612,11 +702,11 @@ async def build_chat_context(
 
     # Use RAG?
     use_rag_val = (str(use_rag).lower() != "false") if use_rag is not None else True
-    if incognito or not allow_tool_preprocessing or is_research_spinoff:
+    if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
         use_rag_val = False
 
     # If pre-fetched search context was provided (compare mode), skip live web search
-    skip_web = bool(search_context) or not allow_tool_preprocessing
+    skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
 
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
@@ -635,7 +725,7 @@ async def build_chat_context(
         incognito=incognito,
         use_skills=skills_enabled,
     )
-    if use_rag is not None or is_research_spinoff:
+    if use_rag is not None or is_research_spinoff or casual_low_signal:
         _preface_kwargs["use_rag"] = use_rag_val
     preface, rag_sources, web_sources = chat_processor.build_context_preface(**_preface_kwargs)
 
@@ -643,7 +733,7 @@ async def build_chat_context(
     used_memories = getattr(chat_processor, '_last_used_memories', [])
 
     # Inject pre-fetched search context (compare mode)
-    if search_context and allow_tool_preprocessing:
+    if search_context and allow_tool_preprocessing and not casual_low_signal:
         preface.append(untrusted_context_message("prefetched search context", search_context))
 
     # YouTube transcripts
@@ -702,6 +792,7 @@ async def build_chat_context(
         preset=preset,
         preprocessed=preprocessed,
         auto_opened_docs=auto_opened_docs,
+        uploaded_files=uploaded_files,
     )
 
 

routes/chat_routes.py

@@ -29,6 +29,7 @@ from routes.document_helpers import _owner_session_filter
 from core.database import SessionLocal, get_session_mode, set_session_mode
 from core.database import Session as DBSession, ChatMessage as DBChatMessage
 from core.database import Document as DBDocument, ModelEndpoint
+from core.log_safety import redact_url
 from routes.research_routes import _resolve_research_endpoint
 from routes.model_routes import _visible_models
 from routes.chat_helpers import (
@@ -728,6 +729,15 @@ def setup_chat_routes(
                         logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}")
                 else:
                     logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}")
+            if not active_doc:
+                _email_doc_q = _doc_db.query(DBDocument).filter(
+                    DBDocument.session_id == session,
+                    DBDocument.is_active == True,
+                    DBDocument.language == "email",
+                )
+                active_doc = _owner_session_filter(_email_doc_q, ctx.user).order_by(DBDocument.updated_at.desc()).first()
+                if active_doc:
+                    logger.info(f"[doc-inject] found email draft by session fallback: title={active_doc.title!r}")
             if not active_doc:
                 _session_doc_q = _doc_db.query(DBDocument).filter(
                     DBDocument.session_id == session,
@@ -789,19 +799,19 @@ def setup_chat_routes(
                 "manage_skills",      # skill presets tied to user
             })
 
-        # Active email reader open → strip the tools that let the agent
-        # "drift" to a new compose: create_document (writes a fake email-
-        # shaped .md file) and send_email (sends fresh to a recipient the
-        # agent invented). With those gone, the only paths left for "write
-        # email saying X" are ui_control open_email_reply (draft) and
-        # reply_to_email (immediate send) — both of which use the open
-        # email's UID. Code-level enforcement instead of relying on a
-        # prompt rule the model can ignore.
+        # Active email reader open → strip the tools that let the agent drift
+        # away from the visible email or skip review. The only allowed compose
+        # path is ui_control open_email_reply, which opens the same draft editor
+        # as the Reply button with the generated body pre-filled. This prevents
+        # the model from falling back to direct SMTP when it botches a draft
+        # call, and prevents fake email-shaped documents.
         if active_email_ctx and active_email_ctx.get("uid"):
             disabled_tools.update({
                 "create_document",
                 "send_email",
+                "reply_to_email",
                 "mcp__email__send_email",
+                "mcp__email__reply_to_email",
             })
 
         # Enforce per-user privileges
@@ -829,7 +839,11 @@ def setup_chat_routes(
         from src.settings import get_setting
         _global_disabled = get_setting("disabled_tools", [])
         if _global_disabled and isinstance(_global_disabled, list):
-            disabled_tools.update(_global_disabled)
+            explicit_web_allowed = allow_web_search is not None and str(allow_web_search).lower() == "true"
+            if explicit_web_allowed:
+                disabled_tools.update(t for t in _global_disabled if t not in {"web_search", "web_fetch"})
+            else:
+                disabled_tools.update(_global_disabled)
 
         # Light auto-escalation: the user is in chat mode and just expressed a
         # notes/calendar/email intent. Grant the relevant managers but withhold
@@ -926,7 +940,7 @@ def setup_chat_routes(
             if effective_do_research:
                 _r_ep, _r_model, _r_headers = _resolve_research_endpoint(sess)
                 _auth_keys = list(_r_headers.keys()) if _r_headers else []
-                logger.info(f"Research endpoint resolved: model={_r_model}, endpoint={_r_ep}, auth_keys={_auth_keys}, sess_headers_keys={list(sess.headers.keys()) if isinstance(sess.headers, dict) else type(sess.headers)}")
+                logger.info(f"Research endpoint resolved: model={_r_model}, endpoint={redact_url(_r_ep)}, auth_keys={_auth_keys}, sess_headers_keys={list(sess.headers.keys()) if isinstance(sess.headers, dict) else type(sess.headers)}")
 
                 # Clarification round: only for very short/vague queries on first research message.
                 # Skip in compare mode — each pane is a fresh session, so every one would
@@ -1250,7 +1264,14 @@ def setup_chat_routes(
                 try:
                     from src.settings import get_setting
                     from src.agent_tools import MAX_AGENT_ROUNDS as _DEFAULT_ROUNDS
-                    _tool_budget = int(get_setting("agent_max_tool_calls", 0))
+                    # Per-message tool budget from settings; guard defensively in
+                    # case settings.json was hand-edited to a non-numeric value
+                    # (the HTTP admin endpoint validates, but direct edits bypass
+                    # it). 0 = unlimited, matching auth_routes set_settings().
+                    try:
+                        _tool_budget = int(get_setting("agent_max_tool_calls", 0))
+                    except (TypeError, ValueError):
+                        _tool_budget = 0
                     # Per-message round cap from settings; clamp defensively in
                     # case settings.json was hand-edited to a bad value.
                     try:
@@ -1259,6 +1280,10 @@ def setup_chat_routes(
                         _max_rounds = _DEFAULT_ROUNDS
                     _max_rounds = max(1, min(_max_rounds, 200))
 
+                    _forced_tools = None
+                    if allow_web_search is not None and str(allow_web_search).lower() == "true":
+                        _forced_tools = {"web_search", "web_fetch"}
+
                     async for chunk in stream_agent_loop(
                         sess.endpoint_url,
                         sess.model,
@@ -1280,6 +1305,8 @@ def setup_chat_routes(
                         plan_mode=plan_mode,
                         approved_plan=approved_plan or None,
                         workspace=workspace or None,
+                        forced_tools=_forced_tools,
+                        uploaded_files=ctx.uploaded_files,
                     ):
                         if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
                             try:
@@ -1332,9 +1359,11 @@ def setup_chat_routes(
                         elif chunk.startswith("event: "):
                             yield chunk
                         elif chunk == "data: [DONE]\n\n":
-                            if full_response:
+                            _has_tool_events = bool((last_metrics or {}).get("tool_events"))
+                            if full_response or _has_tool_events:
+                                _response_to_save = full_response or "Done."
                                 _saved_id = save_assistant_response(
-                                    sess, session_manager, session, full_response, last_metrics,
+                                    sess, session_manager, session, _response_to_save, last_metrics,
                                     character_name=ctx.preset.character_name,
                                     web_sources=web_sources,
                                     rag_sources=ctx.rag_sources,
@@ -1344,7 +1373,7 @@ def setup_chat_routes(
                                 if _saved_id:
                                     yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
                                 run_post_response_tasks(
-                                    sess, session_manager, session, message, full_response,
+                                    sess, session_manager, session, message, _response_to_save,
                                     last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
                                     incognito=incognito, compare_mode=compare_mode,
                                     character_name=ctx.preset.character_name,

routes/codex_routes.py

@@ -15,6 +15,7 @@ from typing import Any
 from fastapi import APIRouter, BackgroundTasks, Body, HTTPException, Request
 from fastapi.responses import StreamingResponse
 
+from core.middleware import require_admin
 from src.auth_helpers import require_authenticated_request, require_user
 from src.tool_implementations import do_manage_notes
 from src.constants import COOKBOOK_STATE_FILE
@@ -109,6 +110,20 @@ def _scope_owner_all(request: Request, required: set[str]) -> str:
     return require_user(request)
 
 
+def _require_cookbook_scope(request: Request, allowed: set[str]) -> str:
+    """Authorize a Codex cookbook route.
+
+    For API-token callers, enforce the given scope set.
+    For cookie-session callers, additionally require admin privileges
+    because cookbook surfaces expose host topology, task logs, tmux
+    commands, and model-serving controls.
+    """
+    owner = _scope_owner(request, allowed)
+    if not getattr(request.state, "api_token", False):
+        require_admin(request)
+    return owner
+
+
 def _find_endpoint(router: APIRouter | None, method: str, path: str):
     if router is None:
         return None
@@ -118,6 +133,18 @@ def _find_endpoint(router: APIRouter | None, method: str, path: str):
     return None
 
 
+def _clamp_pagination(offset: Any, limit: Any, *, default_limit: int = 50, max_limit: int = 50) -> tuple[int, int]:
+    try:
+        parsed_offset = int(0 if offset in (None, "") else offset)
+    except (TypeError, ValueError):
+        raise HTTPException(400, "Invalid offset")
+    try:
+        parsed_limit = int(default_limit if limit in (None, "") else limit)
+    except (TypeError, ValueError):
+        raise HTTPException(400, "Invalid limit")
+    return max(0, parsed_offset), max(1, min(parsed_limit, max_limit))
+
+
 def setup_codex_routes(
     email_router: APIRouter | None = None,
     memory_router: APIRouter | None = None,
@@ -425,10 +452,18 @@ def setup_codex_routes(
         owner = _scope_owner(request, DOCS_READ_SCOPES)
         if documents_library_endpoint is None:
             raise HTTPException(503, "Documents integration is not available")
-        return await _as_owner(
+        offset, limit = _clamp_pagination(offset, limit)
+        result = await _as_owner(
             request, owner, documents_library_endpoint,
             request, search, language, sort, offset, limit, archived,
         )
+        if isinstance(result, dict):
+            docs = result.get("documents")
+            total = result.get("total")
+            if isinstance(docs, list) and isinstance(total, int):
+                next_offset = offset + len(docs)
+                result["next_offset"] = next_offset if next_offset < total else None
+        return result
 
     @router.get("/documents/{doc_id}")
     async def codex_documents_get(request: Request, doc_id: str):
@@ -532,14 +567,14 @@ def setup_codex_routes(
 
     @router.get("/cookbook/tasks")
     async def codex_cookbook_tasks(request: Request):
-        _scope_owner(request, COOKBOOK_READ_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
         state = _read_cookbook_state()
         tasks = state.get("tasks") or []
         return {"tasks": [_redact_task(t) for t in tasks]}
 
     @router.get("/cookbook/servers")
     async def codex_cookbook_servers(request: Request):
-        _scope_owner(request, COOKBOOK_READ_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
         state = _read_cookbook_state()
         servers = state.get("env", {}).get("servers") or []
         # Strip ssh creds / passwords; keep only what's needed to pick a host.
@@ -558,7 +593,7 @@ def setup_codex_routes(
 
     @router.get("/cookbook/output/{session_id}")
     async def codex_cookbook_output(request: Request, session_id: str, tail: int = 400):
-        _scope_owner(request, COOKBOOK_READ_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
         # Defensive: session_id must be the tmux-style id we issue
         # (`serve-XXXX` / `cookbook-XXXX` / `queue-XXXX`); anything else
         # would let the agent run arbitrary `tmux capture-pane` targets.
@@ -600,7 +635,7 @@ def setup_codex_routes(
 
     @router.post("/cookbook/serve")
     async def codex_cookbook_serve(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
-        _scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
         # Wraps /api/model/serve with the SAME validation the UI uses.
         # _validate_serve_cmd (called inside model_serve) rejects shell
         # metachars and requires the leading binary to be in the
@@ -639,7 +674,7 @@ def setup_codex_routes(
 
     @router.post("/cookbook/stop/{session_id}")
     async def codex_cookbook_stop(request: Request, session_id: str):
-        _scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
         import re as _re
         if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id):
             raise HTTPException(400, "Invalid session id")
@@ -659,7 +694,7 @@ def setup_codex_routes(
         """List cached models on a configured server (or local if host is omitted).
         Mirrors `list_cached_models` from the chat agent so external agents have
         the same inventory view before deciding what to serve/download."""
-        _scope_owner(request, COOKBOOK_READ_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
         # Hit /api/model/cached internally, with the same modelDirs the chat
         # agent's list_cached_models would resolve from cookbook state.
         state = _read_cookbook_state()
@@ -721,7 +756,7 @@ def setup_codex_routes(
         """List saved serve presets (model + host + port + launch cmd).
         Counterpart to `list_serve_presets`. Use BEFORE composing a `serve`
         body — the user's saved preset usually has the working cmd already."""
-        _scope_owner(request, COOKBOOK_READ_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
         state = _read_cookbook_state()
         presets = state.get("presets") or []
         out = []
@@ -741,7 +776,7 @@ def setup_codex_routes(
     async def codex_cookbook_serve_preset(request: Request, name: str):
         """Launch a saved preset by name. Reuses the working cmd + host the
         user already saved, avoiding the cmd-allowlist trial-and-error loop."""
-        _scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
         import re as _re
         if not _re.fullmatch(r"[A-Za-z0-9 _.:@\-]+", name):
             raise HTTPException(400, "Invalid preset name")
@@ -793,7 +828,7 @@ def setup_codex_routes(
         cookbook tracking. Needed when serve_model rejects a cmd and the
         agent falls back to direct ssh — without adoption the session is
         invisible to the UI. Body: {tmux_session, model, host?, port?}."""
-        _scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
         norm = dict(body or {})
         sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip()
         model = (norm.get("model") or norm.get("repo_id") or "").strip()

routes/contacts_routes.py

@@ -18,6 +18,7 @@ from pathlib import Path
 from datetime import datetime
 from urllib.parse import urljoin, urlparse, urlunparse
 
+from core.log_safety import redact_url
 from fastapi import APIRouter, Query, Depends, Response, HTTPException
 from typing import List, Dict, Optional
 
@@ -149,6 +150,14 @@ def _vunesc(value: str) -> str:
 
 def _parse_vcards(text: str) -> List[Dict]:
     """Parse a stream of vCards into dicts with name, email, phone."""
+    # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
+    # space or tab is a continuation of the previous logical line. Real
+    # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
+    # PHOTO lines, and splitting on raw newlines without unfolding dropped the
+    # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
+    # email/name.
+    text = re.sub(r"\r\n[ \t]", "", text or "")
+    text = re.sub(r"\n[ \t]", "", text)
     contacts = []
     for block in re.split(r"BEGIN:VCARD", text):
         if not block.strip():
@@ -689,15 +698,24 @@ def _delete_contact(uid: str) -> bool:
         url = _resolve_resource_url(uid)
         auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
         r = httpx.delete(url, auth=auth, timeout=10)
-        if r.status_code in (200, 204):
-            _contact_cache["fetched_at"] = None
-            return True
-        if r.status_code == 404:
-            # Resource not found at the resolved URL. With href resolution
-            # this should be rare (genuinely already deleted). Invalidate
-            # the cache and report success so the UI doesn't keep a ghost.
-            logger.info(f"CardDAV DELETE 404 for {uid} — treating as already gone")
+        if r.status_code in (200, 204, 404):
+            # Invalidate cache so the next fetch sees the server truth.
             _contact_cache["fetched_at"] = None
+            # Verify: force a fresh fetch and check the UID is actually gone.
+            # A 404 on the guessed URL ({uid}.vcf) can mean the contact
+            # lives at a different resource URL — the DELETE missed it but
+            # we'd silently report success. This check catches that.
+            fresh = _fetch_contacts(force=True)
+            still_there = any(c.get("uid") == uid for c in fresh)
+            if still_there:
+                logger.warning(
+                    f"CardDAV DELETE reported success for {uid} "
+                    f"but UID still present after re-fetch — "
+                    f"resource URL may differ from {redact_url(url)}"
+                )
+                return False
+            if r.status_code == 404:
+                logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
             return True
         logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
         return False

routes/cookbook_helpers.py

@@ -558,10 +558,22 @@ def _bash_squote(v: str) -> str:
     return v.replace("'", "'\\''")
 
 
+# Shown by generated runner scripts when the ollama binary is missing on the
+# target host. Must stay free of backticks/$( ) and be emitted single-quoted:
+# an earlier version wrapped the install one-liner in backticks inside a
+# double-quoted echo, which bash executed as command substitution and ran the
+# system-wide installer (including on remote SSH hosts) instead of printing
+# the hint.
+OLLAMA_MISSING_HINT = (
+    "ERROR: Ollama not found on this server. Install it from "
+    "https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh"
+)
+
+
 # Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
 # Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
 _SERVE_CMD_ALLOWLIST = {
-    "vllm", "llama-server", "llama_server", "llama.cpp", "ollama",
+    "vllm", "llama-server", "llama-server.exe", "llama_server", "llama.cpp", "ollama",
     "python", "python3",
     "sglang", "lmdeploy",
     "node", "npx",
@@ -577,6 +589,16 @@ _SERVE_CMD_ALLOWLIST = {
 _GGUF_PRELUDE_RE = re.compile(
     r'^MODEL_FILE=\$\([^\n]*?\)\s*&&\s*\{[^{}]*\}\s*\|\|\s*\{[^{}]*\}\s*&&\s*'
 )
+_SAFE_SUBSHELL_TEXT = r"[^'\n;&|`$()<>]+"
+_SAFE_SUBSHELL_DQ_HOME_PATH = r'"\$HOME/[^"\n;&|`()<>]*"'
+_SAFE_PRINTF_SUBSHELL_RE = re.compile(
+    rf"^\$\(printf[ \t]+%s[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|\$\{{HOME\}}'/{_SAFE_SUBSHELL_TEXT}')\)$"
+)
+_SAFE_FIND_MMPROJ_SUBSHELL_RE = re.compile(
+    rf"^\$\(find[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|{_SAFE_SUBSHELL_DQ_HOME_PATH}|{_SAFE_SUBSHELL_TEXT})"
+    r"[ \t]+-iname[ \t]+'mmproj\*\.gguf'"
+    r"(?:[ \t]+2>/dev/null)?[ \t]*\|[ \t]*sort[ \t]*\|[ \t]*head[ \t]+-1\)$"
+)
 _OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)")
 _OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$")
 _OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
@@ -677,6 +699,13 @@ def _check_serve_binary(seg: str) -> None:
         )
 
 
+def _is_safe_serve_subshell(subshell: str) -> bool:
+    return bool(
+        _SAFE_PRINTF_SUBSHELL_RE.fullmatch(subshell)
+        or _SAFE_FIND_MMPROJ_SUBSHELL_RE.fullmatch(subshell)
+    )
+
+
 def _validate_serve_cmd(v: str | None) -> str | None:
     """Reject serve commands that aren't in the allowlist or contain shell metachars.
 
@@ -708,15 +737,15 @@ def _validate_serve_cmd(v: str | None) -> str | None:
             _check_serve_binary(part.strip())
         return v
 
-    # Otherwise: a single invocation — no shell metacharacters allowed.
-    # Temporarily replace safe $(printf %s ...) expressions with a placeholder
-    # to avoid triggering the metacharacter/command-injection checks.
-    cleaned_v = v
-    printf_matches = list(re.finditer(r"\$\(\s*printf\s+%s\s+([^\n()]*?)\)", v))
-    for match in printf_matches:
-        inner = match.group(1)
-        if not any(c in inner for c in (";", "&&", "||", "$(", "`")):
-            cleaned_v = cleaned_v.replace(match.group(0), "/placeholder/safe/path.gguf")
+    # Otherwise: a single invocation — no shell metacharacters allowed. Replace
+    # only the exact command substitutions emitted by the Cookbook UI:
+    # $(printf %s 'safe-path') and the mmproj lookup
+    # $(find <path> -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1).
+    def _replace_safe_subshell(match: re.Match[str]) -> str:
+        subshell = match.group(0)
+        return "/placeholder/safe/path" if _is_safe_serve_subshell(subshell) else subshell
+
+    cleaned_v = re.sub(r"\$\([^()]*\)", _replace_safe_subshell, v)
 
     # (`$(` was the original intent; bare `$` is fine for shell-safe paths.)
     if any(c in cleaned_v for c in (";", "&&", "||", "$(")):
@@ -786,25 +815,149 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
     to hard-wire CUDA on Linux. That made ROCm hosts attempt a CUDA configure and
     fail with "CUDA Toolkit not found" instead of building with HIP.
     """
+    # Try a prebuilt binary from llama.cpp's GitHub releases FIRST — no
+    # cmake/build-essential/git/CUDA-headers needed at all. The from-source
+    # build below stays as a fallback (custom flags, esoteric arch, no
+    # internet, etc). 30 seconds vs 5+ minutes of compile, and removes
+    # every OS-package dep from the launch path. Sets _odysseus_have_prebuilt=1
+    # on success; the existing build-tier if/elif chain below is gated on
+    # that variable so we never compile twice or shadow the prebuilt symlink.
+    runner_lines.append('    _odysseus_have_prebuilt=""')
+    runner_lines.append('    _odysseus_arch="$(uname -m)"')
+    runner_lines.append('    _odysseus_prebuilt_url=""')
+    runner_lines.append('    if command -v curl >/dev/null 2>&1 && [ "$_odysseus_arch" = "x86_64" ]; then')
+    runner_lines.append('      _odysseus_pat=""')
+    runner_lines.append('      _odysseus_has_nv_inline() { command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU "; }')
+    runner_lines.append('      _odysseus_has_vk_inline() { ldconfig -p 2>/dev/null | grep -q "libvulkan\\.so" || command -v vulkaninfo >/dev/null 2>&1 || [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ]; }')
+    runner_lines.append('      _odysseus_has_vkdev_inline() { ls /dev/dri/renderD* >/dev/null 2>&1 || (lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\'); }')
+    runner_lines.append('      if _odysseus_has_nv_inline; then')
+    runner_lines.append('        _odysseus_pat="ubuntu.*cuda"')
+    runner_lines.append('      elif _odysseus_has_vkdev_inline && _odysseus_has_vk_inline; then')
+    runner_lines.append('        _odysseus_pat="ubuntu.*vulkan"')
+    runner_lines.append('      else')
+    runner_lines.append('        _odysseus_pat="ubuntu-x64\\\\.zip"')
+    runner_lines.append('      fi')
+    runner_lines.append('      _odysseus_prebuilt_url="$(curl -fsSL --max-time 15 https://api.github.com/repos/ggml-org/llama.cpp/releases/latest 2>/dev/null | grep \'"browser_download_url"\' | cut -d\'"\' -f4 | grep -iE "$_odysseus_pat" | grep -iv "arm\\|aarch64" | head -1)"')
+    runner_lines.append('    fi')
+    # Accept any of unzip / bsdtar / python3 -m zipfile as the extractor.
+    # python3 is essentially always present on modern Linux, so this lets
+    # the prebuilt path work on minimal Ubuntu installs that lack `unzip`.
+    runner_lines.append('    if [ -n "$_odysseus_prebuilt_url" ] && (command -v unzip >/dev/null 2>&1 || command -v bsdtar >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1); then')
+    runner_lines.append('      echo "[odysseus] Found prebuilt llama-server: $_odysseus_prebuilt_url"')
+    runner_lines.append('      mkdir -p ~/bin "$HOME/.cache/odysseus/llama-cpp-prebuilt" && cd "$HOME/.cache/odysseus/llama-cpp-prebuilt"')
+    runner_lines.append('      rm -f llama-cpp.zip')
+    runner_lines.append('      if curl -fsSL --max-time 120 "$_odysseus_prebuilt_url" -o llama-cpp.zip && [ -s llama-cpp.zip ]; then')
+    runner_lines.append('        rm -rf build && mkdir -p build')
+    runner_lines.append('        if command -v unzip >/dev/null 2>&1; then unzip -qq -o llama-cpp.zip -d build; elif command -v bsdtar >/dev/null 2>&1; then bsdtar -xf llama-cpp.zip -C build; else python3 -c "import zipfile; zipfile.ZipFile(\\"llama-cpp.zip\\").extractall(\\"build\\")"; fi')
+    runner_lines.append('        _odysseus_extracted="$(find build -type f -name llama-server 2>/dev/null | head -1)"')
+    runner_lines.append('        if [ -n "$_odysseus_extracted" ]; then')
+    runner_lines.append('          chmod +x "$_odysseus_extracted"')
+    runner_lines.append('          ln -sf "$_odysseus_extracted" ~/bin/llama-server')
+    runner_lines.append('          _odysseus_libdir="$(dirname "$_odysseus_extracted")"')
+    runner_lines.append('          mkdir -p ~/.config && echo "export LD_LIBRARY_PATH=\\"$_odysseus_libdir:\\${LD_LIBRARY_PATH:-}\\"" > ~/.config/odysseus-llama-cpp-env')
+    runner_lines.append('          _odysseus_have_prebuilt=1')
+    runner_lines.append('          echo "[odysseus] Prebuilt llama-server installed at $_odysseus_extracted"')
+    runner_lines.append('        fi')
+    runner_lines.append('      fi')
+    runner_lines.append('      [ -z "$_odysseus_have_prebuilt" ] && echo "[odysseus] Prebuilt download/extract failed — falling back to from-source build."')
+    runner_lines.append('    elif [ -z "$_odysseus_prebuilt_url" ]; then')
+    runner_lines.append('      echo "[odysseus] No matching prebuilt llama-server for this host (arch=$_odysseus_arch) — will build from source."')
+    runner_lines.append('    fi')
+    runner_lines.append('  if [ -z "$_odysseus_have_prebuilt" ]; then')
     # Detect pip-installed nvcc (from vLLM/nvidia CUDA wheels) and put it on PATH
-    # so cmake's CUDA configure can find it. We keep this after the ROCm/HIP
-    # check — a machine with both stacks should honor the native HIP toolchain on
-    # AMD hosts instead of accidentally preferring a stray nvcc wheel.
-    runner_lines.append('    for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do')
-    runner_lines.append('      [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break')
-    runner_lines.append('    done')
+    # so cmake's CUDA configure can find it — BUT only when actual NVIDIA
+    # hardware is present. On AMD/Intel hosts the pip nvcc is a misleading
+    # leftover (no libcudart, no GPU it could target) and would otherwise
+    # send the build down the CUDA branch and fail with "CUDA Toolkit not
+    # found" instead of trying Vulkan.
+    runner_lines.append('    _odysseus_has_nvidia_hw() {')
+    runner_lines.append('      command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU " && return 0')
+    runner_lines.append('      ls /dev/nvidia* >/dev/null 2>&1 && return 0')
+    runner_lines.append('      lspci 2>/dev/null | grep -iE \'VGA|3D|Display\' | grep -iq nvidia && return 0')
+    runner_lines.append('      return 1')
+    runner_lines.append('    }')
+    runner_lines.append('    if _odysseus_has_nvidia_hw; then')
+    runner_lines.append('      for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do')
+    runner_lines.append('        [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break')
+    runner_lines.append('      done')
+    runner_lines.append('    fi')
     # rm -rf build so a prior poisoned CMakeCache.txt (e.g. from a failed CUDA
     # or HIP attempt) doesn't cause the next configure to reuse stale settings.
     runner_lines.append('    mkdir -p ~/bin')
-    runner_lines.append('    cd ~/llama.cpp && rm -rf build')
+    # Try to install cmake / build-essential / git automatically before the
+    # build, but ONLY via passwordless sudo (`sudo -n`) — interactive sudo
+    # would hang a tmux-backgrounded serve task waiting for a password. If
+    # sudo asks for a password the install is skipped silently and the
+    # diagnosis pattern (cookbook_routes.py / cookbook_helpers.py) surfaces
+    # an explicit "install cmake" suggestion in the Cookbook diagnosis
+    # toolbar after the inevitable build failure.
+    runner_lines.append('    _odysseus_apt_bootstrap() {')
+    runner_lines.append('      local _missing=""')
+    runner_lines.append('      command -v cmake >/dev/null 2>&1 || _missing="$_missing cmake"')
+    runner_lines.append('      command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _missing="$_missing build-essential"')
+    runner_lines.append('      command -v git >/dev/null 2>&1 || _missing="$_missing git"')
+    runner_lines.append('      [ -z "$_missing" ] && return 0')
+    runner_lines.append('      if command -v apt-get >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
+    runner_lines.append('        echo "[odysseus] Auto-installing missing build deps via apt:$_missing"')
+    runner_lines.append('        sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>&1 | tail -3')
+    runner_lines.append('        sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $_missing 2>&1 | tail -5 || true')
+    runner_lines.append('      elif command -v pacman >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
+    runner_lines.append('        echo "[odysseus] Auto-installing missing build deps via pacman:$_missing"')
+    runner_lines.append('        local _pacpkgs="$(echo "$_missing" | sed -e \'s/build-essential/base-devel/g\')"')
+    runner_lines.append('        sudo -n pacman -Sy --needed --noconfirm $_pacpkgs 2>&1 | tail -5 || true')
+    runner_lines.append('      elif command -v dnf >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
+    runner_lines.append('        echo "[odysseus] Auto-installing missing build deps via dnf:$_missing"')
+    runner_lines.append('        local _dnfpkgs="$(echo "$_missing" | sed -e \'s/build-essential/gcc gcc-c++ make/g\')"')
+    runner_lines.append('        sudo -n dnf install -y $_dnfpkgs 2>&1 | tail -5 || true')
+    runner_lines.append('      else')
+    runner_lines.append('        echo "[odysseus] WARNING: missing build deps ($_missing) — passwordless sudo is unavailable, cannot auto-install. Cookbook Diagnosis will explain the fix after the build fails."')
+    runner_lines.append('      fi')
+    runner_lines.append('    }')
+    runner_lines.append('    _odysseus_apt_bootstrap')
+    runner_lines.append('    _odysseus_missing_build_deps=""')
+    runner_lines.append('    command -v cmake >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps cmake"')
+    runner_lines.append('    command -v git >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps git"')
+    runner_lines.append('    command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps build-essential"')
+    runner_lines.append('    if [ -n "$_odysseus_missing_build_deps" ]; then')
+    runner_lines.append('      echo "ERROR: llama.cpp source build needs missing packages:$_odysseus_missing_build_deps"')
+    runner_lines.append('      if command -v apt-get >/dev/null 2>&1; then')
+    runner_lines.append('        echo "Install on this host: sudo apt-get update && sudo apt-get install -y cmake build-essential git"')
+    runner_lines.append('      elif command -v pacman >/dev/null 2>&1; then')
+    runner_lines.append('        echo "Install on this host: sudo pacman -Sy --needed cmake base-devel git"')
+    runner_lines.append('      elif command -v dnf >/dev/null 2>&1; then')
+    runner_lines.append('        echo "Install on this host: sudo dnf install -y cmake gcc gcc-c++ make git"')
+    runner_lines.append('      fi')
+    runner_lines.append('      echo "Alternative: install a native llama-server on PATH, then relaunch."')
+    runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
+    runner_lines.append('    fi')
+    runner_lines.append('    cd ~/llama.cpp')
+    runner_lines.append('    _odysseus_has_vulkan() {')
+    runner_lines.append('      ldconfig -p 2>/dev/null | grep -q \'libvulkan\\.so\' && return 0')
+    runner_lines.append('      [ -e /usr/lib/libvulkan.so.1 ] && return 0')
+    runner_lines.append('      [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ] && return 0')
+    runner_lines.append('      command -v vulkaninfo >/dev/null 2>&1 && return 0')
+    runner_lines.append('      return 1')
+    runner_lines.append('    }')
+    runner_lines.append('    _odysseus_has_vulkan_device() {')
+    runner_lines.append('      ls /dev/dri/renderD* >/dev/null 2>&1 && return 0')
+    runner_lines.append('      lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\' && return 0')
+    runner_lines.append('      return 1')
+    runner_lines.append('    }')
+    # Backend preference: native ROCm/HIP > native CUDA > Vulkan > CPU.
+    # Vulkan is a portable fallback that works on AMD when ROCm isn't
+    # installed (e.g. Strix Halo) and on any vendor's discrete GPU, but
+    # it's ~30-40% slower than native HIP/CUDA for LLM inference — only
+    # pick it when no native toolchain is present.
     runner_lines.append('    if command -v hipconfig &>/dev/null || [ -d /opt/rocm ] || [ -n "$ROCM_PATH" ] || [ -n "$HIP_PATH" ]; then')
+    runner_lines.append('      rm -rf build')
     runner_lines.append('      if command -v hipconfig &>/dev/null; then')
     runner_lines.append('        export HIPCXX="${HIPCXX:-$(hipconfig -l)/clang}"')
     runner_lines.append('        export HIP_PATH="${HIP_PATH:-$(hipconfig -R)}"')
     runner_lines.append('      fi')
     runner_lines.append('      echo "[odysseus] ROCm/HIP detected — building llama-server with HIP support..."')
     runner_lines.append('      cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_HIP=ON && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
-    runner_lines.append('    elif command -v nvcc &>/dev/null; then')
+    runner_lines.append('    elif command -v nvcc &>/dev/null && _odysseus_has_nvidia_hw; then')
+    runner_lines.append('      rm -rf build')
     # nvcc alone is not sufficient — pip-installed CUDA wheels or incomplete
     # tooling can expose nvcc without shipping libcudart, causing cmake to fail
     # mid-build with "CUDA runtime library not found". Check cudart explicitly
@@ -828,31 +981,50 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
     runner_lines.append('        echo "[odysseus]   Ensure libcudart is installed (e.g. cuda-runtime package) and visible via ldconfig or CUDA_HOME."')
     runner_lines.append('        cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
     runner_lines.append('      fi')
+    runner_lines.append('    elif _odysseus_has_vulkan_device && _odysseus_has_vulkan; then')
+    runner_lines.append('      echo "[odysseus] Vulkan-capable GPU detected (no ROCm/CUDA toolchain installed) — building llama-server with Vulkan support..."')
+    runner_lines.append('      rm -rf build-vulkan')
+    runner_lines.append('      cmake -B build-vulkan -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON && cmake --build build-vulkan -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build-vulkan/bin/llama-server ~/bin/llama-server')
     runner_lines.append('    else')
-    runner_lines.append('      echo "[odysseus] WARNING: no HIP/CUDA toolchain found — building llama-server for CPU only."')
+    runner_lines.append('      echo "[odysseus] WARNING: no HIP/CUDA/Vulkan toolchain found — building llama-server for CPU only."')
     runner_lines.append('      echo "[odysseus]   GPU inference will not be available for this llama.cpp build."')
-    runner_lines.append('      echo "[odysseus]   Install ROCm for AMD GPUs or vLLM/CUDA tooling for NVIDIA, then re-launch this serve task."')
+    runner_lines.append('      echo "[odysseus]   Install Vulkan (libvulkan-dev) / ROCm for AMD GPUs or CUDA tooling for NVIDIA, then re-launch this serve task."')
+    runner_lines.append('      rm -rf build')
     runner_lines.append('      cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
     runner_lines.append('    fi')
+    runner_lines.append('  fi  # end _odysseus_have_prebuilt guard')
 
 
-def _llama_cpp_rebuild_cmd() -> str:
+def _llama_cpp_rebuild_cmd(update_source: bool = False) -> str:
     """Shell command that clears the Cookbook-managed llama.cpp build.
 
-    Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build``
+    Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build*``
     directory so the next llama.cpp serve recompiles from source, picking up a
     CUDA or HIP toolchain if one is now available. The serve bootstrap only
     builds when ``llama-server`` is missing from PATH, so without this an
-    existing CPU-only build is reused forever. It deliberately installs and
-    downloads nothing; the rebuild itself happens on the next serve.
+    existing CPU-only build is reused forever. When ``update_source`` is true,
+    the command also fast-forwards the Cookbook-managed ``~/llama.cpp`` checkout
+    if it exists. The rebuild itself happens on the next serve.
     """
+    update_cmd = ''
+    if update_source:
+        update_cmd = (
+            'if [ -d "$HOME/llama.cpp/.git" ]; then '
+            'git -C "$HOME/llama.cpp" pull --ff-only --depth 1 || '
+            'echo "[odysseus] WARNING: llama.cpp source update failed; clearing cached build anyway."; '
+            'elif command -v git >/dev/null 2>&1; then '
+            'git clone --depth 1 https://github.com/ggml-org/llama.cpp "$HOME/llama.cpp" || '
+            'echo "[odysseus] WARNING: llama.cpp clone failed; clearing cached build anyway."; '
+            'fi && '
+        )
     return (
         'mkdir -p "$HOME/bin" && '
+        f'{update_cmd}'
         'rm -f "$HOME/bin/llama-server" && '
-        'rm -rf "$HOME/llama.cpp/build" && '
+        'rm -rf "$HOME/llama.cpp/build" "$HOME/llama.cpp/build-vulkan" && '
         'echo "[odysseus] Cleared the cached llama.cpp build. '
         'Re-launch the serve task to rebuild llama-server from source '
-        '(CUDA or HIP will be used if a toolchain is now available)."'
+        '(Vulkan, HIP, or CUDA will be used if a matching toolchain is now available)."'
     )
 
 
@@ -1115,8 +1287,27 @@ def _diagnose_serve_output(text: str) -> dict | None:
             "SGLang is not installed or not in PATH on this server.",
             [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
         ),
+        # System build deps come BEFORE the generic llama.cpp catch-all so
+        # cmake / build-essential / git missing → a specific OS-package
+        # remediation instead of "install llama-cpp-python[server]" (which
+        # itself fails to compile when cmake is absent).
         (
-            r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'|git: command not found|cmake: command not found",
+            r"cmake: command not found|cmake.*not found.*[Cc]ould not",
+            "cmake is required to build llama.cpp from source but isn't installed on this server.",
+            [{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}],
+        ),
+        (
+            r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler",
+            "A C/C++ compiler (build-essential) is required to build llama.cpp from source.",
+            [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
+        ),
+        (
+            r"^git: command not found",
+            "git is required to clone the llama.cpp source tree.",
+            [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
+        ),
+        (
+            r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'",
             "llama.cpp / llama-cpp-python dependencies are missing.",
             [{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}],
         ),

routes/document_helpers.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel
 
 from core.database import Document, DocumentVersion
 from core.database import Session as DbSession
+from src.auth_helpers import _auth_disabled
 from src.upload_handler import UploadHandler
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,7 @@ class DocumentCreate(BaseModel):
 class DocumentUpdate(BaseModel):
     content: str
     summary: Optional[str] = None
+    force_version: bool = False
 
 class DocumentPatch(BaseModel):
     title: Optional[str] = None
@@ -78,6 +80,8 @@ def _verify_doc_owner(db, doc: Document, user: str):
     the session join for any not-yet-backfilled legacy row.
     """
     if user is None:
+        if _auth_disabled():
+            return  # Single-user / no-auth mode: allow access
         raise HTTPException(403, "Authentication required")
     if doc.owner is not None:
         if doc.owner != user:
@@ -104,7 +108,6 @@ def _owner_session_filter(q, user):
     by the time this filter is live there are no NULL-owner rows to leak;
     we therefore match the owner strictly for authenticated callers."""
     if not user:
-        from src.auth_helpers import _auth_disabled
         if user == "" or _auth_disabled():
             return q
         return q.filter(False)

routes/document_routes.py

@@ -10,7 +10,7 @@ from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File,
 from sqlalchemy import case, func, or_
 from core.database import SessionLocal, Document, DocumentVersion
 from core.database import Session as DbSession
-from src.auth_helpers import get_current_user
+from src.auth_helpers import get_current_user, _auth_disabled
 from src.constants import MAIL_ATTACHMENTS_DIR
 
 logger = logging.getLogger(__name__)
@@ -388,7 +388,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
         db = SessionLocal()
         try:
             if not user:
-                raise HTTPException(403, "Authentication required")
+                if not _auth_disabled():
+                    raise HTTPException(403, "Authentication required")
             # v2 review HIGH-9: raise 403 explicitly when the caller
             # can't see this session, instead of returning [] which the
             # UI treats identically to "no docs" and silently masks
@@ -569,8 +570,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 raise HTTPException(404, "Document not found")
             _verify_doc_owner(db, doc, user)
 
-            # Skip if content is identical
-            if doc.current_content == req.content:
+            # Skip if content is identical unless the caller explicitly wants
+            # a checkpoint version from the current editor state.
+            if doc.current_content == req.content and not req.force_version:
                 return _doc_to_dict(doc)
 
             _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
@@ -582,7 +584,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
 
             now = datetime.now(timezone.utc)
             coalesced = False
-            if latest_ver and latest_ver.source == "user":
+            if latest_ver and latest_ver.source == "user" and not req.force_version:
                 ver_time = latest_ver.created_at
                 if ver_time.tzinfo is None:
                     ver_time = ver_time.replace(tzinfo=timezone.utc)
@@ -798,10 +800,26 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
             from src.document_actions import _JUNK_TITLES
 
             to_delete = []
+            now = datetime.now(timezone.utc)
             for doc in docs:
+                created = doc.created_at
+                if created and created.tzinfo is None:
+                    created = created.replace(tzinfo=timezone.utc)
+
+                # Skip freshly created documents to avoid deleting them while the user is actively editing
+                if created and (now - created).total_seconds() < 900:  # 15 minutes
+                    continue
+
                 content = (doc.current_content or "").strip()
                 title_raw = (doc.title or "").strip()
                 title = title_raw.lower()
+                is_fresh_empty = (
+                    not content
+                    and created is not None
+                    and (now - created).total_seconds() < 1800
+                )
+                if is_fresh_empty:
+                    continue
 
                 # Strip markdown noise to get a "real" character count
                 stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE)
@@ -836,10 +854,6 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                     to_delete.append(doc); deleted += 1; continue
                 if title in _JUNK_TITLES:
                     to_delete.append(doc); deleted += 1; continue
-                if real_len < 30:
-                    to_delete.append(doc); deleted += 1; continue
-                if "\n" not in content and real_len < 50:
-                    to_delete.append(doc); deleted += 1; continue
 
                 # Fix empty or placeholder titles on survivors
                 if not title_raw or title_raw == "Untitled":

routes/email_helpers.py

@@ -40,6 +40,16 @@ from src.secret_storage import decrypt as _decrypt
 logger = logging.getLogger(__name__)
 
 
+class EmailNotConfiguredError(RuntimeError):
+    """Raised when an IMAP operation is attempted on an account that has no
+    inbox configured (e.g. a send-only / SMTP-only account).
+
+    Subclasses RuntimeError so existing broad ``except Exception`` handlers
+    keep working; callers that want to treat "no inbox" as an empty result
+    rather than a failure can catch this type specifically.
+    """
+
+
 def _xoauth2_raw(user: str, access_token: str) -> str:
     """The SASL XOAUTH2 initial-response string (unencoded).
 
@@ -225,8 +235,9 @@ def _strip_think(text: str) -> str:
     """
     if not text:
         return ""
-    from src.text_helpers import strip_think as _central, _THINK_CLOSED_RE, _THINK_OPEN_RE, _THINK_TAG_RE
-    had_think = bool(_THINK_CLOSED_RE.search(text) or _THINK_OPEN_RE.search(text) or _THINK_TAG_RE.search(text))
+    from src.text_helpers import strip_think as _central, _THINK_TAG_RE
+    # Single linear tag check; the old closed/open `.search()` calls could ReDoS.
+    had_think = bool(_THINK_TAG_RE.search(text))
     return _central(text, prose=had_think, prompt_echo=True)
 
 
@@ -413,12 +424,19 @@ SCHEDULED_DB = Path(SCHEDULED_EMAILS_DB)
 OWNER_SCOPED_EMAIL_CACHE_TABLES = {
     "email_summaries",
     "email_ai_replies",
+    "email_translations",
     "email_calendar_extractions",
     "email_urgency_alerts",
     "sender_signatures",
 }
 
 
+def email_translation_body_hash(body: str) -> str:
+    import hashlib as _hashlib
+    normalized = (body or "").strip()
+    return _hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest()
+
+
 def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
     owner = (owner or "").strip()
     if owner:
@@ -426,14 +444,34 @@ def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
     return "(owner = '' OR owner IS NULL)", ()
 
 
-def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, columns: list[str]):
+def _ensure_owner_scoped_email_cache_table(
+    conn,
+    table: str,
+    create_sql: str,
+    columns: list[str],
+    pk_columns: list[str] | None = None,
+):
     """Rebuild legacy Message-ID-only cache tables with owner in the PK."""
+    desired_pk_cols = pk_columns or ["message_id", "owner"]
     conn.execute(create_sql)
     try:
         info = conn.execute(f"PRAGMA table_info({table})").fetchall()
         cols = [r[1] for r in info]
         pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
-        if "owner" in cols and pk_cols == ["message_id", "owner"]:
+        for col in columns:
+            if col not in cols:
+                if col == "owner":
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN owner TEXT DEFAULT ''")
+                elif col in {"event_uids"}:
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT '[]'")
+                elif col.startswith("has_") or col.endswith("_created") or col.endswith("_count"):
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} INTEGER DEFAULT 0")
+                elif col == "created_at":
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT ''")
+                else:
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT")
+                cols.append(col)
+        if "owner" in cols and pk_cols == desired_pk_cols:
             return
 
         conn.execute(f"ALTER TABLE {table} RENAME TO {table}__old")
@@ -566,6 +604,25 @@ def _init_scheduled_db():
             PRIMARY KEY (message_id, owner)
         )
     """, ["message_id", "owner", "uid", "folder", "reply", "model_used", "created_at"])
+    _ensure_owner_scoped_email_cache_table(conn, "email_translations", """
+        CREATE TABLE IF NOT EXISTS email_translations (
+            body_hash TEXT,
+            owner TEXT DEFAULT '',
+            target_language TEXT DEFAULT 'English',
+            uid TEXT,
+            folder TEXT,
+            subject TEXT,
+            sender TEXT,
+            translation TEXT,
+            same_language INTEGER DEFAULT 0,
+            model_used TEXT,
+            created_at TEXT NOT NULL,
+            PRIMARY KEY (body_hash, owner, target_language)
+        )
+    """, [
+        "body_hash", "owner", "target_language", "uid", "folder", "subject", "sender",
+        "translation", "same_language", "model_used", "created_at",
+    ], ["body_hash", "owner", "target_language"])
     # Email tags / spam classification cache. SECURITY: keyed by
     # (message_id, owner) because Message-IDs are GLOBAL (a newsletter goes
     # to many users with the same Message-ID). Without owner-scoping, a
@@ -575,6 +632,7 @@ def _init_scheduled_db():
         CREATE TABLE IF NOT EXISTS email_tags (
             message_id TEXT,
             owner TEXT DEFAULT '',
+            account_id TEXT DEFAULT '',
             uid TEXT,
             folder TEXT,
             subject TEXT,
@@ -585,7 +643,7 @@ def _init_scheduled_db():
             moved_to TEXT,
             model_used TEXT,
             created_at TEXT NOT NULL,
-            PRIMARY KEY (message_id, owner)
+            PRIMARY KEY (message_id, owner, account_id)
         )
     """)
     # Backfill migration: older installs created the table with
@@ -593,28 +651,35 @@ def _init_scheduled_db():
     # promote it into the PK by rebuild-copy-swap (SQLite can't ALTER PK).
     try:
         _cols = [r[1] for r in conn.execute("PRAGMA table_info(email_tags)")]
+        _pk_cols = [r[1] for r in sorted(conn.execute("PRAGMA table_info(email_tags)").fetchall(), key=lambda row: row[5] or 99) if r[5]]
         if "owner" not in _cols:
-            # Add the column first so reads/writes don't break mid-migration.
             conn.execute("ALTER TABLE email_tags ADD COLUMN owner TEXT DEFAULT ''")
-            # Rebuild with composite PK. Existing rows get owner='' (legacy
-            # single-user); the urgency scanner will overwrite as it
-            # re-classifies. No data loss.
+            _cols.append("owner")
+        if "account_id" not in _cols:
+            conn.execute("ALTER TABLE email_tags ADD COLUMN account_id TEXT DEFAULT ''")
+            _cols.append("account_id")
+        if _pk_cols != ["message_id", "owner", "account_id"]:
+            # Rebuild with account-aware composite PK. Existing rows get
+            # account_id='' and are still readable as legacy fallback rows;
+            # fresh task runs write exact account ids and no longer block each
+            # other when two accounts share a Message-ID.
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS email_tags__new (
                     message_id TEXT,
                     owner TEXT DEFAULT '',
+                    account_id TEXT DEFAULT '',
                     uid TEXT, folder TEXT, subject TEXT, sender TEXT,
                     tags TEXT, spam_verdict INTEGER DEFAULT 0,
                     spam_reason TEXT, moved_to TEXT, model_used TEXT,
                     created_at TEXT NOT NULL,
-                    PRIMARY KEY (message_id, owner)
+                    PRIMARY KEY (message_id, owner, account_id)
                 )
             """)
             conn.execute("""
                 INSERT OR IGNORE INTO email_tags__new
-                  (message_id, owner, uid, folder, subject, sender, tags,
+                  (message_id, owner, account_id, uid, folder, subject, sender, tags,
                    spam_verdict, spam_reason, moved_to, model_used, created_at)
-                SELECT message_id, COALESCE(owner, ''), uid, folder, subject,
+                SELECT message_id, COALESCE(owner, ''), COALESCE(account_id, ''), uid, folder, subject,
                        sender, tags, spam_verdict, spam_reason, moved_to,
                        model_used, created_at
                 FROM email_tags
@@ -630,11 +695,12 @@ def _init_scheduled_db():
             message_id TEXT,
             owner TEXT DEFAULT '',
             uid TEXT,
+            event_uids TEXT DEFAULT '[]',
             events_created INTEGER DEFAULT 0,
             created_at TEXT NOT NULL,
             PRIMARY KEY (message_id, owner)
         )
-    """, ["message_id", "owner", "uid", "events_created", "created_at"])
+    """, ["message_id", "owner", "uid", "event_uids", "events_created", "created_at"])
     _ensure_owner_scoped_email_cache_table(conn, "email_urgency_alerts", """
         CREATE TABLE IF NOT EXISTS email_urgency_alerts (
             message_id TEXT,
@@ -660,6 +726,64 @@ def _init_scheduled_db():
             PRIMARY KEY (owner, account_key, folder, message_key)
         )
     """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS email_message_index (
+            owner TEXT NOT NULL DEFAULT '',
+            account_key TEXT NOT NULL DEFAULT '',
+            folder TEXT NOT NULL,
+            uid TEXT NOT NULL,
+            message_id TEXT,
+            subject TEXT,
+            from_name TEXT,
+            from_address TEXT,
+            to_text TEXT,
+            cc_text TEXT,
+            date_iso TEXT,
+            date_display TEXT,
+            date_epoch REAL DEFAULT 0,
+            size INTEGER DEFAULT 0,
+            flags TEXT DEFAULT '',
+            has_attachments INTEGER DEFAULT 0,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (owner, account_key, folder, uid)
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
+        ON email_message_index(owner, account_key, folder, date_epoch DESC)
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS ix_email_message_index_message_id
+        ON email_message_index(owner, account_key, message_id)
+    """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS email_body_preview_cache (
+            owner TEXT NOT NULL DEFAULT '',
+            account_key TEXT NOT NULL DEFAULT '',
+            folder TEXT NOT NULL,
+            uid TEXT NOT NULL,
+            message_id TEXT,
+            payload_json TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (owner, account_key, folder, uid)
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS ix_email_body_preview_message_id
+        ON email_body_preview_cache(owner, account_key, message_id)
+    """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS email_attachment_metadata_cache (
+            owner TEXT NOT NULL DEFAULT '',
+            account_key TEXT NOT NULL DEFAULT '',
+            folder TEXT NOT NULL,
+            uid TEXT NOT NULL,
+            message_id TEXT,
+            attachments_json TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (owner, account_key, folder, uid)
+        )
+    """)
     # Boundary cache — LLM-detected sig/quote start positions in the body.
     # Stored as char offsets (-1 = no boundary found). Once cached, the
     # client uses these to fold without ever re-calling the LLM.
@@ -928,6 +1052,14 @@ def _imap_connect(account_id: str | None = None, owner: str = "",
     # `timeout` is overridable so short-lived callers (e.g. the service-health
     # probe) can impose a tighter budget than the default IMAP timeout.
     cfg = _get_email_config(account_id, owner=owner)
+    # Send-only (SMTP-only) account: no IMAP host means there is no inbox to
+    # read. Bail out with a clear, typed error instead of handing an empty
+    # host to imaplib — IMAP4("", 993) silently dials localhost:993 and fails
+    # with a confusing "[Errno 111] Connection refused" on every inbox poll.
+    if not cfg.get("imap_host"):
+        raise EmailNotConfiguredError(
+            f"IMAP is not configured for account {cfg.get('account_name') or 'default'!r}"
+        )
     # Connection mode:
     #   STARTTLS on → plain + upgrade
     #   STARTTLS off + port 993 → implicit SSL (IMAPS)
@@ -1141,10 +1273,15 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str
     try:
         c = _imap_connect(account_id, owner=owner)
         c.select(_q(src))
-        status, _ = c.copy(uid, _q(dest))
+        # Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy()
+        # and store() operate on message SEQUENCE NUMBERS, so addressing them
+        # with a UID moved/deleted the wrong message (or silently no-oped when
+        # the UID exceeded the message count). Use the UID commands, matching
+        # the move/delete path in email_routes.py.
+        status, _ = c.uid("COPY", uid, _q(dest))
         if status != "OK":
             return False
-        c.store(uid, "+FLAGS", "\\Deleted")
+        c.uid("STORE", uid, "+FLAGS", "\\Deleted")
         c.expunge()
         return True
     except Exception as e:
@@ -1233,56 +1370,95 @@ def _list_attachments_from_msg(msg):
         return attachments
     idx = 0
     for part in msg.walk():
-        if part.is_multipart():
-            continue
         cd = str(part.get("Content-Disposition", ""))
         ct = part.get_content_type()
+        is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename())
+        if part.is_multipart() and not is_attached_email:
+            continue
         # Skip text/html body parts (only consider real attachments)
         if ct in ("text/plain", "text/html") and "attachment" not in cd:
             continue
         filename = part.get_filename()
         if filename:
             filename = _decode_header(filename)
+            if ct == "message/rfc822" and not re.search(r"\.[A-Za-z0-9]{1,8}$", filename):
+                filename = f"{filename}.eml"
         else:
             # Inline images, etc. - generate a name
-            ext = ct.split("/")[-1] if "/" in ct else "bin"
+            ext = "eml" if ct == "message/rfc822" else (ct.split("/")[-1] if "/" in ct else "bin")
             filename = f"attachment_{idx}.{ext}"
         payload = part.get_payload(decode=True)
-        size = len(payload) if payload else 0
+        if payload is None and ct == "message/rfc822":
+            try:
+                payload = part.as_bytes()
+            except Exception:
+                payload = b""
+        size = len(payload) if payload is not None else 0
+        content_id = (part.get("Content-ID") or "").strip().strip("<>")
         attachments.append({
             "index": idx,
             "filename": filename,
             "content_type": ct,
             "size": size,
             "is_inline": "inline" in cd.lower(),
+            "content_id": content_id,
         })
         idx += 1
     return attachments
 
 
+def _is_likely_signature_image_attachment(att: dict) -> bool:
+    """Match the reader's inline signature/logo image filter."""
+    filename = str((att or {}).get("filename") or "").lower()
+    if not re.search(r"\.(png|jpe?g|gif|bmp|svg|webp)$", filename):
+        return False
+    size = int((att or {}).get("size") or 0)
+    if re.search(r"^image\d{3,}\.(png|jpe?g|gif)$", filename):
+        return True
+    if re.search(r"^(signature|logo|sig|footer|banner)[-_\d]*\.(png|jpe?g|gif|svg)$", filename):
+        return True
+    return 0 < size < 30 * 1024
+
+
+def _has_visible_attachments(msg) -> bool:
+    """Return True only for attachments the reader will render as chips."""
+    return any(
+        not _is_likely_signature_image_attachment(att)
+        for att in _list_attachments_from_msg(msg)
+    )
+
+
 def _extract_attachment_to_disk(msg, index, target_dir):
     """Extract a specific attachment to disk and return the file path."""
     if not msg.is_multipart():
         return None
     idx = 0
     for part in msg.walk():
-        if part.is_multipart():
-            continue
         cd = str(part.get("Content-Disposition", ""))
         ct = part.get_content_type()
+        is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename())
+        if part.is_multipart() and not is_attached_email:
+            continue
         if ct in ("text/plain", "text/html") and "attachment" not in cd:
             continue
         if idx == index:
             filename = part.get_filename()
             if filename:
                 filename = _decode_header(filename)
+                if ct == "message/rfc822" and not re.search(r"\.[A-Za-z0-9]{1,8}$", filename):
+                    filename = f"{filename}.eml"
             else:
-                ext = ct.split("/")[-1] if "/" in ct else "bin"
+                ext = "eml" if ct == "message/rfc822" else (ct.split("/")[-1] if "/" in ct else "bin")
                 filename = f"attachment_{idx}.{ext}"
             # Sanitize
             safe_name = re.sub(r"[^\w\s\-.]", "_", filename).strip()
             payload = part.get_payload(decode=True)
-            if not payload:
+            if payload is None and ct == "message/rfc822":
+                try:
+                    payload = part.as_bytes()
+                except Exception:
+                    payload = b""
+            if payload is None:
                 return None
             target_dir.mkdir(parents=True, exist_ok=True)
             filepath = target_dir / safe_name
@@ -1664,6 +1840,10 @@ class SendEmailRequest(BaseModel):
     attachments: Optional[List[str]] = None
     # Which account to send from. None = default account.
     account_id: Optional[str] = None
+    # Source message for replies. When present, /send marks this exact message
+    # answered after successful delivery so it leaves undone/reply-soon views.
+    source_uid: Optional[str] = None
+    source_folder: Optional[str] = None
     # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
     odysseus_kind: Optional[str] = None
     # If true, /send waits for SMTP + Sent append and returns the sent UID.

routes/email_pollers.py

@@ -29,7 +29,7 @@ from datetime import datetime
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
-from src.llm_core import llm_call_async
+from src.task_endpoint import resolve_task_candidates, task_llm_call_async
 
 from routes.email_helpers import (
     _strip_think, _extract_reply, _apply_email_style_mechanics, _load_settings, _save_settings, _get_email_config,
@@ -44,6 +44,46 @@ from routes.email_helpers import (
 
 logger = logging.getLogger(__name__)
 
+# Recovers a `[{"action": ...}, ...]` JSON array from raw LLM output when the
+# fenced-block strip leaves nothing usable. Runs on model output influenced by
+# untrusted email bodies, so it must not backtrack: the object content class is
+# `[^{}]` (brace-delimited, greedy) rather than the old `[^[\]]*?` lazy runs,
+# which exploded exponentially on inputs like `[{"action"},{` + `}},{{` * N
+# (CodeQL py/redos #198).
+_CAL_ACTION_ARRAY_RE = re.compile(
+    r'\[\s*\{[^{}]*"action"[^{}]*\}\s*(?:,\s*\{[^{}]*\}\s*)*\]',
+    re.DOTALL,
+)
+
+
+def _extract_json_array_from_text(text: str):
+    """Return the last valid JSON array embedded in model output, if any."""
+    if not text:
+        return None
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip(), flags=re.MULTILINE).strip()
+    decoder = json.JSONDecoder()
+    try:
+        parsed = decoder.decode(cleaned)
+        if isinstance(parsed, list):
+            return parsed
+    except Exception:
+        pass
+
+    # Models often explain themselves and finish with `[]` or `[{"action":...}]`.
+    # Scan every array opener and keep the last complete JSON array, rather than
+    # using a greedy regex that can swallow prose containing square brackets.
+    last = None
+    for idx, ch in enumerate(cleaned):
+        if ch != "[":
+            continue
+        try:
+            parsed, _end = decoder.raw_decode(cleaned[idx:])
+        except Exception:
+            continue
+        if isinstance(parsed, list):
+            last = parsed
+    return last
+
 
 def _owner_for_email_account(account_id: str | None) -> str:
     if not account_id:
@@ -77,6 +117,8 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
                                    do_tag: bool = False, do_spam: bool = False,
                                    do_calendar: bool = False,
                                    days_back: int = 1,
+                                   account_id: str | None = None,
+                                   max_process: int | None = None,
                                    progress_cb=None) -> str:
     """One iteration of the email scan. Temporarily flips settings flags
     so the existing background-loop logic runs exactly once for the requested ops."""
@@ -91,7 +133,12 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
     settings["email_auto_calendar"] = bool(do_calendar)
     _save_settings(settings)
     try:
-        return await _auto_summarize_pass(days_back=days_back, progress_cb=progress_cb)
+        return await _auto_summarize_pass(
+            days_back=days_back,
+            account_id=account_id,
+            max_process=max_process,
+            progress_cb=progress_cb,
+        )
     finally:
         s2 = _load_settings()
         for k, v in prev.items():
@@ -129,7 +176,7 @@ def _latest_inbox_fallback_uids(conn, reconnect):
         return [], reconnect()
 
 
-async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
     """Single pass of the auto-summarize/reply scan.
 
     When account_id is None, iterates over every enabled account in
@@ -156,28 +203,41 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
             names = {}
         if len(ids) <= 1:
             # Single-account (or zero rows — fallback to legacy settings.json lookup)
-            return await _auto_summarize_pass_single(days_back=days_back, account_id=(ids[0] if ids else None), progress_cb=progress_cb)
+            return await _auto_summarize_pass_single(
+                days_back=days_back,
+                account_id=(ids[0] if ids else None),
+                max_process=max_process,
+                progress_cb=progress_cb,
+            )
         outs = []
         for idx, aid in enumerate(ids, start=1):
             try:
                 await _emit_progress(progress_cb, f"{names.get(aid, aid[:8])}: starting ({idx}/{len(ids)})")
-                result = await _auto_summarize_pass_single(days_back=days_back, account_id=aid, progress_cb=progress_cb)
+                result = await _auto_summarize_pass_single(
+                    days_back=days_back,
+                    account_id=aid,
+                    max_process=max_process,
+                    progress_cb=progress_cb,
+                )
                 outs.append(f"[{names.get(aid, aid[:8])}] {result}")
             except Exception as e:
                 logger.warning(f"auto-summarize pass failed for account {aid}: {e}")
                 outs.append(f"[{names.get(aid, aid[:8])}] error: {e}")
         return "\n".join(outs)
-    return await _auto_summarize_pass_single(days_back=days_back, account_id=account_id, progress_cb=progress_cb)
+    return await _auto_summarize_pass_single(
+        days_back=days_back,
+        account_id=account_id,
+        max_process=max_process,
+        progress_cb=progress_cb,
+    )
 
 
-async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
     """Single pass of the auto-summarize/reply scan for ONE account.
     Reads current settings flags."""
     import asyncio
     import sqlite3 as _sql3
-    import requests as _req
-    from src.endpoint_resolver import resolve_endpoint
-    from src.llm_core import _uses_max_completion_tokens, _restricts_temperature
+    from src.llm_core import _uses_max_completion_tokens
 
     settings = _load_settings()
     auto_sum = settings.get("email_auto_summarize", False)
@@ -254,9 +314,15 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         ).fetchall()}
         if auto_tag or auto_spam:
             if account_owner:
-                _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner=?", (account_owner,)).fetchall()}
+                _tag_existing = {r[0] for r in _c.execute(
+                    "SELECT message_id FROM email_tags WHERE owner=? AND (account_id=? OR account_id='' OR account_id IS NULL)",
+                    (account_owner, account_id or ""),
+                ).fetchall()}
             else:
-                _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner='' OR owner IS NULL").fetchall()}
+                _tag_existing = {r[0] for r in _c.execute(
+                    "SELECT message_id FROM email_tags WHERE (owner='' OR owner IS NULL) AND (account_id=? OR account_id='' OR account_id IS NULL)",
+                    (account_id or "",),
+                ).fetchall()}
         else:
             _tag_existing = set()
         _cal_existing = {r[0] for r in _c.execute(
@@ -285,11 +351,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         if auto_spam and not spam_folder:
             logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move")
 
-        url, model, headers = resolve_endpoint("utility", owner=account_owner)
-        if not url:
-            url, model, headers = resolve_endpoint("default", owner=account_owner)
-        if not url or not model:
+        task_candidates = resolve_task_candidates(owner=account_owner)
+        if not task_candidates:
             return "No model configured"
+        url, model, headers = task_candidates[0]
 
         writing_style = settings.get("email_writing_style", "")
         processed = 0
@@ -303,7 +368,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         _reply_failed = 0
         _detail_lines = []
         _current_folder = "INBOX"
-        _max_process = 5
+        # Calendar extraction is sequential and each row can involve a model
+        # call plus a calendar write. Keep the scheduled calendar-only pass
+        # below the 5-minute action budget instead of timing out mid-run.
+        _default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5
+        try:
+            _max_process = max(1, int(max_process)) if max_process is not None else _default_max_process
+        except Exception:
+            _max_process = _default_max_process
         for _entry in uid_list:
             if processed >= _max_process:
                 break
@@ -395,48 +467,30 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     req_headers.update(headers)
 
                 if need_sum:
-                    tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
-                    payload = {
-                        "model": model,
-                        "messages": [
-                            {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
-                            {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
-                        ],
-                        tok_key: 16384,
-                        "temperature": 0.3,
-                        "stream": False,
-                    }
-                    # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
-                    if _restricts_temperature(model):
-                        payload.pop("temperature", None)
                     try:
-                        # Use to_thread so this sync HTTP call doesn't freeze
-                        # the entire event loop while the LLM thinks (240s).
-                        resp = await asyncio.to_thread(
-                            _req.post, url, json=payload, headers=req_headers, timeout=240
+                        summary = await task_llm_call_async(
+                            messages=[
+                                {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
+                                {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
+                            ],
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.3, max_tokens=16384, timeout=240,
                         )
-                        if resp.ok:
-                            rdata = resp.json()
-                            m = (rdata.get("choices") or [{}])[0].get("message", {})
-                            summary = (m.get("content") or "").strip()
-                            summary = _extract_reply(summary)
-                            if not summary:
-                                rc = (m.get("reasoning_content") or "").strip()
-                                bullets = [ln.strip() for ln in rc.split("\n") if re.match(r"^[-•*]\s+|^\d+[.)]\s+", ln.strip())]
-                                summary = "\n".join(bullets) if bullets else ""
-                            if summary:
-                                _c = _sql3.connect(SCHEDULED_DB)
-                                _c.execute("""
-                                    INSERT OR REPLACE INTO email_summaries
-                                    (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
-                                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-                                """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
-                                _c.commit()
-                                _c.close()
-                                _sum_existing.add(message_id)
-                                _summaries_created += 1
-                                _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
-                                _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
+                        summary = _extract_reply((summary or "").strip())
+                        if summary:
+                            _c = _sql3.connect(SCHEDULED_DB)
+                            _c.execute("""
+                                INSERT OR REPLACE INTO email_summaries
+                                (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
+                                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
+                            _c.commit()
+                            _c.close()
+                            _sum_existing.add(message_id)
+                            _summaries_created += 1
+                            _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                            _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
                     except Exception as e:
                         _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                         _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
@@ -457,14 +511,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     if context_snippets:
                         sys_prompt += "\n\nRELEVANT CONTEXT FROM PAST EMAILS AND CONTACTS:\n" + "\n\n---\n\n".join(context_snippets[:5])
                     try:
-                        reply = await llm_call_async(
-                            url=url, model=model,
+                        reply = await task_llm_call_async(
                             messages=[
                                 {"role": "system", "content": sys_prompt},
                                 {"role": "user", "content": f"Original email:\nFrom: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\nDraft a reply. Return only the reply body text."},
                             ],
-                            temperature=0.7, max_tokens=1024,
-                            headers=req_headers, timeout=90,
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.7, max_tokens=1024, timeout=90,
                         )
                         reply = _apply_email_style_mechanics(_extract_reply(reply or ""))
                         if reply:
@@ -491,6 +545,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                 # ── Calendar event extraction (independent of reply drafting) ──
                 if need_cal:
                     _cal_run_count = 0
+                    _cal_event_uids = []
+                    _cal_parse_ok = False
                     try:
                         # Pull a snapshot of upcoming events so the LLM can decide
                         # create vs update vs cancel based on what already exists.
@@ -499,8 +555,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                         _existing_summary = get_upcoming_events(_acct_owner, horizon_days=60, limit=40)
                         existing_json = json.dumps(_existing_summary)
                         is_sent = _folder.lower().startswith("sent") or "sent" in _folder.lower()
-                        cal_extract = await llm_call_async(
-                            url=url, model=model,
+                        cal_extract = await task_llm_call_async(
                             messages=[
                                 {"role": "system", "content": (
                                     "You are a calendar assistant. The user receives emails AND sends replies "
@@ -551,21 +606,22 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                     f"{body[:4000]}"
                                 )},
                             ],
-                            temperature=0.1, max_tokens=16384,
-                            headers=req_headers, timeout=180,
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.1, max_tokens=16384, timeout=75,
                         )
                         _raw_original = cal_extract or ""
                         cal_extract = _strip_think(_raw_original)
                         cal_extract = re.sub(r"^```(?:json)?\s*|\s*```$", "", cal_extract, flags=re.MULTILINE).strip()
                         if not cal_extract and _raw_original:
-                            matches = list(re.finditer(r'\[\s*\{[^[\]]*?"action"[^[\]]*?\}\s*(?:,\s*\{[^[\]]*?\}\s*)*\]', _raw_original, re.DOTALL))
+                            matches = list(_CAL_ACTION_ARRAY_RE.finditer(_raw_original))
                             if matches:
                                 cal_extract = matches[-1].group()
                         logger.info(f"[cal-extract] uid={uid.decode() if isinstance(uid, bytes) else uid} folder={_folder} subj={subject[:50]!r} raw_len={len(cal_extract)} orig_len={len(_raw_original)} raw={cal_extract[:800]!r}")
-                        jm = re.search(r'\[.*\]', cal_extract, re.DOTALL)
-                        if jm:
+                        ops = _extract_json_array_from_text(cal_extract)
+                        if ops is not None:
                             try:
-                                ops = json.loads(jm.group())
+                                _cal_parse_ok = True
                                 logger.info(f"[cal-extract] parsed {len(ops)} op(s)")
                                 if isinstance(ops, list) and ops:
                                     from src.tool_implementations import do_manage_calendar
@@ -595,6 +651,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                             r = await do_manage_calendar(json.dumps(args), owner=_acct_owner)
                                             if r.get("exit_code", 0) == 0:
                                                 logger.info(f"[cal-extract] Updated event uid={cuid} → {op.get('title')} {op['date']}")
+                                                if cuid and cuid not in _cal_event_uids:
+                                                    _cal_event_uids.append(cuid)
                                                 _cal_run_count += 1
                                             else:
                                                 logger.warning(f"[cal-extract] update failed: {r.get('error')}")
@@ -675,28 +733,43 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                             r = await do_manage_calendar(cal_args, owner=_acct_owner)
                                             if r.get("exit_code", 0) == 0:
                                                 logger.info(f"[cal-extract] Created event: {op['title']} on {op['date']}")
+                                                _created_uid = (r.get("uid") or "").strip()
+                                                if _created_uid and _created_uid not in _cal_event_uids:
+                                                    _cal_event_uids.append(_created_uid)
                                                 _events_created += 1
                                                 _cal_run_count += 1
                                             else:
                                                 logger.warning(f"[cal-extract] create failed: {r.get('error')} args={cal_args[:200]}")
                             except Exception as je:
                                 logger.warning(f"[cal-extract] JSON parse failed: {je} on raw={cal_extract[:200]!r}")
+                        else:
+                            logger.warning(f"[cal-extract] no JSON array found on raw={cal_extract[:200]!r}")
                     except Exception as e:
                         logger.warning(f"[cal-extract] Meeting extraction LLM call failed for uid={uid}: {e}")
-                    # Record we processed this email so we don't re-LLM next run
-                    try:
-                        _cc = _sql3.connect(SCHEDULED_DB)
-                        _cc.execute(
-                            "INSERT OR REPLACE INTO email_calendar_extractions "
-                            "(message_id, owner, uid, events_created, created_at) VALUES (?, ?, ?, ?, ?)",
-                            (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid),
-                             _cal_run_count, datetime.utcnow().isoformat())
-                        )
-                        _cc.commit()
-                        _cc.close()
-                        _cal_existing.add(message_id)
-                    except Exception as ce:
-                        logger.debug(f"Could not cache calendar extraction: {ce}")
+                    else:
+                        # Record successfully parsed results so we don't re-LLM
+                        # no-op emails. Transient LLM failures are retried on
+                        # the next poll run.
+                        try:
+                            if _cal_parse_ok:
+                                _cc = _sql3.connect(SCHEDULED_DB)
+                                _cc.execute(
+                                    "INSERT OR REPLACE INTO email_calendar_extractions "
+                                    "(message_id, owner, uid, event_uids, events_created, created_at) VALUES (?, ?, ?, ?, ?, ?)",
+                                    (
+                                        message_id,
+                                        account_owner or "",
+                                        uid.decode() if isinstance(uid, bytes) else str(uid),
+                                        json.dumps(_cal_event_uids),
+                                        _cal_run_count,
+                                        datetime.utcnow().isoformat(),
+                                    ),
+                                )
+                                _cc.commit()
+                                _cc.close()
+                                _cal_existing.add(message_id)
+                        except Exception as ce:
+                            logger.debug(f"Could not cache calendar extraction: {ce}")
 
                 if need_urgent:
                     try:
@@ -728,9 +801,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             "temperature": 0,
                             tok_key: 200,
                         }
-                        urg_raw = await llm_call_async(
-                            url=url, model=model, messages=payload["messages"],
-                            temperature=0, max_tokens=200, headers=req_headers, timeout=60,
+                        urg_raw = await task_llm_call_async(
+                            messages=payload["messages"],
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0, max_tokens=200, timeout=60,
                         )
                         urg_raw = _strip_think(urg_raw or "")
                         urg_raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", urg_raw, flags=re.MULTILINE).strip()
@@ -831,8 +906,13 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                         class_sys = (
                             "Classify the email. Return ONLY a JSON object, no prose, no markdown fences. "
                             "Schema: {\"tags\": [\"tag1\"], \"spam\": false, \"reason\": \"short\"}. "
-                            "Pick 1-2 tags from: work, personal, finance, bills, receipt, travel, "
-                            "newsletter, promo, notification, security, social, shopping, calendar.\n\n"
+                            "Pick 1-3 tags from: work, personal, urgent, action-needed, finance, bills, "
+                            "receipt, legal, travel, newsletter, promo, notification, security, social, "
+                            "shopping, calendar, support.\n\n"
+                            "Use work for professional/company/client/operations messages. "
+                            "Use personal for friends/family/private-life messages. "
+                            "Use urgent for real time-sensitive consequences. "
+                            "Use action-needed when the user likely needs to reply, pay, sign, book, or decide.\n\n"
                             "Set spam=true for ANY of:\n"
                             "- Phishing, scams, chain mail, deceptive offers\n"
                             "- Marketing/promotional blasts (\"special offer\", \"limited time\", discount codes)\n"
@@ -849,70 +929,55 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             "If it's a mass-mailed generic update with no personal CTA, mark spam=true even if from a legitimate service. "
                             "Reason should be 5-10 words."
                         )
-                        tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
-                        payload = {
-                            "model": model,
-                            "messages": [
+                        raw_out = await task_llm_call_async(
+                            messages=[
                                 {"role": "system", "content": class_sys},
                                 {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body[:4000]}"},
                             ],
-                            tok_key: 512,
-                            "temperature": 0.1,
-                            "stream": False,
-                        }
-                        # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
-                        if _restricts_temperature(model):
-                            payload.pop("temperature", None)
-                        # to_thread keeps the event loop responsive during the LLM call
-                        resp = await asyncio.to_thread(
-                            _req.post, url, json=payload, headers=req_headers, timeout=120
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.1, max_tokens=512, timeout=120,
                         )
-                        if not resp.ok:
-                            logger.warning(f"Auto-classify {uid.decode() if isinstance(uid, bytes) else str(uid)} HTTP {resp.status_code}: {resp.text[:200]}")
-                        else:
-                            rdata = resp.json()
-                            m = (rdata.get("choices") or [{}])[0].get("message", {})
-                            raw_out = (m.get("content") or "").strip()
-                            raw_out = _strip_think(raw_out)
-                            raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
-                            jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
-                            parsed = None
-                            if jm:
-                                try:
-                                    parsed = json.loads(jm.group(0))
-                                except Exception:
-                                    parsed = None
-                            if parsed is not None:
-                                _ALLOWED_TAGS = {"work","personal","finance","bills","receipt","travel",
-                                                 "newsletter","marketing","notification","security","social",
-                                                 "shopping","calendar"}
-                                raw_tags = parsed.get("tags") or []
-                                if isinstance(raw_tags, str):
-                                    raw_tags = [raw_tags]
-                                tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
-                                tags = ["marketing" if t == "promo" else t for t in tags]
-                                tags = [t for t in tags if t in _ALLOWED_TAGS][:2]
-                                is_spam = bool(parsed.get("spam"))
-                                spam_reason = str(parsed.get("reason") or "")[:200]
+                        raw_out = _strip_think((raw_out or "").strip())
+                        raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
+                        jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
+                        parsed = None
+                        if jm:
+                            try:
+                                parsed = json.loads(jm.group(0))
+                            except Exception:
+                                parsed = None
+                        if parsed is not None:
+                            _ALLOWED_TAGS = {"work","personal","urgent","action-needed","finance","bills",
+                                             "receipt","legal","travel","newsletter","marketing","notification",
+                                             "security","social","shopping","calendar","support"}
+                            raw_tags = parsed.get("tags") or []
+                            if isinstance(raw_tags, str):
+                                raw_tags = [raw_tags]
+                            tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
+                            tags = ["marketing" if t == "promo" else t for t in tags]
+                            tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
+                            is_spam = bool(parsed.get("spam"))
+                            spam_reason = str(parsed.get("reason") or "")[:200]
 
-                                moved_to = ""
-                                if is_spam and auto_spam and spam_folder:
-                                    if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
-                                        moved_to = spam_folder
-                                        logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
+                            moved_to = ""
+                            if is_spam and auto_spam and spam_folder:
+                                if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
+                                    moved_to = spam_folder
+                                    logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
 
-                                _c = _sql3.connect(SCHEDULED_DB)
-                                _c.execute("""
-                                    INSERT OR REPLACE INTO email_tags
-                                    (message_id, owner, uid, folder, subject, sender, tags, spam_verdict,
-                                     spam_reason, moved_to, model_used, created_at)
-                                    VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?, ?, ?)
-                                """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), subject, sender,
-                                      json.dumps(tags), 1 if is_spam else 0,
-                                      spam_reason, moved_to, model, datetime.utcnow().isoformat()))
-                                _c.commit()
-                                _c.close()
-                                _tag_existing.add(message_id)
+                            _c = _sql3.connect(SCHEDULED_DB)
+                            _c.execute("""
+                                INSERT OR REPLACE INTO email_tags
+                                (message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict,
+                                 spam_reason, moved_to, model_used, created_at)
+                                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """, (message_id, account_owner or "", account_id or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender,
+                                  json.dumps(tags), 1 if is_spam else 0,
+                                  spam_reason, moved_to, model, datetime.utcnow().isoformat()))
+                            _c.commit()
+                            _c.close()
+                            _tag_existing.add(message_id)
                     except Exception as e:
                         logger.warning(f"Auto-classify {uid} failed: {e}")
 

routes/gallery/__init__.py

@@ -0,0 +1,6 @@
+"""Gallery route domain package (slice 2a, #4082/#4071).
+
+Contains gallery_routes.py and gallery_helpers.py, migrated from the flat
+routes/ directory. Backward-compat shims at routes/gallery_routes.py and
+routes/gallery_helpers.py re-export from here.
+"""

routes/gallery/gallery_helpers.py

@@ -0,0 +1,145 @@
+"""gallery_helpers.py — extracted helpers, models, and small utilities.
+
+Imported by gallery_routes.py."""
+
+"""Gallery routes — browsable library for photos and AI-generated images."""
+
+import logging
+from datetime import datetime
+from typing import Dict, Any, Optional
+
+from pydantic import BaseModel
+
+from core.database import GalleryImage
+from src.auth_helpers import _auth_disabled
+
+logger = logging.getLogger(__name__)
+
+
+# ---- Request schemas ----
+
+class GalleryPatch(BaseModel):
+    tags: Optional[str] = None
+    favorite: Optional[bool] = None
+    album_id: Optional[str] = None
+
+
+# ---- EXIF extraction ----
+
+def _extract_exif(content: bytes) -> dict:
+    """Extract EXIF metadata from image bytes. Returns dict of fields."""
+    result = {"width": None, "height": None}
+    try:
+        from PIL import Image
+        from io import BytesIO
+        img = Image.open(BytesIO(content))
+        # Read the raw EXIF before any transpose: exif_transpose strips the
+        # orientation tag and with it the parsed EXIF view.
+        exif = img._getexif() if hasattr(img, '_getexif') else None
+
+        # Record DISPLAY dimensions (EXIF-rotated), matching upload_handler.
+        # A phone photo with Orientation 6/8 is stored landscape but shown
+        # portrait, so the raw width/height swap the aspect ratio.
+        try:
+            from PIL import ImageOps
+            img = ImageOps.exif_transpose(img) or img
+        except Exception:
+            pass
+        result["width"] = img.width
+        result["height"] = img.height
+
+        if not exif:
+            return result
+
+        # EXIF tag IDs
+        # 271=Make, 272=Model, 306=DateTime, 36867=DateTimeOriginal
+        # 34853=GPSInfo
+        result["camera_make"] = str(exif.get(271, "")).strip() or None
+        result["camera_model"] = str(exif.get(272, "")).strip() or None
+
+        # Date taken
+        for tag_id in (36867, 36868, 306):  # DateTimeOriginal, DateTimeDigitized, DateTime
+            raw = exif.get(tag_id)
+            if raw:
+                try:
+                    result["taken_at"] = datetime.strptime(str(raw).strip(), "%Y:%m:%d %H:%M:%S")
+                    break
+                except (ValueError, TypeError):
+                    pass
+
+        # GPS
+        gps_info = exif.get(34853)
+        if gps_info and isinstance(gps_info, dict):
+            try:
+                def _to_deg(vals):
+                    d, m, s = [float(v) for v in vals]
+                    return d + m / 60 + s / 3600
+                if 2 in gps_info and 4 in gps_info:
+                    lat = _to_deg(gps_info[2])
+                    lng = _to_deg(gps_info[4])
+                    if gps_info.get(1) == 'S': lat = -lat
+                    if gps_info.get(3) == 'W': lng = -lng
+                    result["gps_lat"] = f"{lat:.6f}"
+                    result["gps_lng"] = f"{lng:.6f}"
+            except Exception:
+                pass
+    except Exception as e:
+        # User-visible failure (photo loses metadata): surface at WARNING
+        # and record on the result so the upload endpoint can pass it back.
+        logger.warning(f"EXIF extraction failed: {e}")
+        result["exif_error"] = str(e)
+    return result
+
+
+# ---- Helpers ----
+
+def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any]:
+    return {
+        "id": img.id,
+        "filename": img.filename,
+        "url": f"/api/generated-image/{img.filename}",
+        "prompt": img.prompt,
+        "caption": img.caption or "",
+        "model": img.model,
+        "size": img.size,
+        "quality": img.quality,
+        "tags": img.tags or "",
+        "ai_tags": img.ai_tags or "",
+        "user_tags": img.tags or "",
+        "session_id": img.session_id,
+        "session_name": session_name,
+        "album_id": img.album_id,
+        "is_active": img.is_active,
+        "favorite": img.favorite or False,
+        "taken_at": img.taken_at.isoformat() if img.taken_at else None,
+        "camera": f"{img.camera_make or ''} {img.camera_model or ''}".strip() or None,
+        "gps": {"lat": img.gps_lat, "lng": img.gps_lng} if img.gps_lat else None,
+        "width": img.width,
+        "height": img.height,
+        "file_size": img.file_size,
+        "created_at": img.created_at.isoformat() if img.created_at else None,
+        "updated_at": img.updated_at.isoformat() if img.updated_at else None,
+    }
+
+
+def _owner_filter(q, user, model_cls=GalleryImage):
+    """Apply owner filtering to a gallery query.
+
+    ``get_current_user`` returns None both in auth-disabled single-user mode
+    and when auth is enabled but no current user was resolved. Preserve the
+    single-user behavior, but fail closed for auth-enabled null-user states.
+    """
+    if user is not None:
+        return q.filter(model_cls.owner == user)
+    if _auth_disabled():
+        return q
+    return q.filter(False)
+
+
+
+def _human_size(nbytes):
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if abs(nbytes) < 1024:
+            return f"{nbytes:.1f} {unit}"
+        nbytes /= 1024
+    return f"{nbytes:.1f} PB"

routes/gallery_helpers.py

@@ -1,144 +1,14 @@
-"""gallery_helpers.py — extracted helpers, models, and small utilities.
+"""Backward-compat shim - canonical location is routes/gallery/gallery_helpers.py.
 
-Imported by gallery_routes.py."""
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.gallery_helpers``, ``from routes.gallery_helpers import X``,
+``importlib.import_module("routes.gallery_helpers")``, and
+``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the same
+object. Keeps existing import paths working after slice 2a (#4082/#4071).
+"""
 
-"""Gallery routes — browsable library for photos and AI-generated images."""
+import sys as _sys
 
-import logging
-from datetime import datetime
-from typing import Dict, Any, Optional
+from routes.gallery import gallery_helpers as _canonical  # noqa: F401
 
-from pydantic import BaseModel
-
-from core.database import GalleryImage
-from src.auth_helpers import _auth_disabled
-
-logger = logging.getLogger(__name__)
-
-
-# ---- Request schemas ----
-
-class GalleryPatch(BaseModel):
-    tags: Optional[str] = None
-    favorite: Optional[bool] = None
-    album_id: Optional[str] = None
-
-
-# ---- EXIF extraction ----
-
-def _extract_exif(content: bytes) -> dict:
-    """Extract EXIF metadata from image bytes. Returns dict of fields."""
-    result = {"width": None, "height": None}
-    try:
-        from PIL import Image
-        from io import BytesIO
-        img = Image.open(BytesIO(content))
-        # Read the raw EXIF before any transpose: exif_transpose strips the
-        # orientation tag and with it the parsed EXIF view.
-        exif = img._getexif() if hasattr(img, '_getexif') else None
-
-        # Record DISPLAY dimensions (EXIF-rotated), matching upload_handler.
-        # A phone photo with Orientation 6/8 is stored landscape but shown
-        # portrait, so the raw width/height swap the aspect ratio.
-        try:
-            from PIL import ImageOps
-            img = ImageOps.exif_transpose(img) or img
-        except Exception:
-            pass
-        result["width"] = img.width
-        result["height"] = img.height
-
-        if not exif:
-            return result
-
-        # EXIF tag IDs
-        # 271=Make, 272=Model, 306=DateTime, 36867=DateTimeOriginal
-        # 34853=GPSInfo
-        result["camera_make"] = str(exif.get(271, "")).strip() or None
-        result["camera_model"] = str(exif.get(272, "")).strip() or None
-
-        # Date taken
-        for tag_id in (36867, 36868, 306):  # DateTimeOriginal, DateTimeDigitized, DateTime
-            raw = exif.get(tag_id)
-            if raw:
-                try:
-                    result["taken_at"] = datetime.strptime(str(raw).strip(), "%Y:%m:%d %H:%M:%S")
-                    break
-                except (ValueError, TypeError):
-                    pass
-
-        # GPS
-        gps_info = exif.get(34853)
-        if gps_info and isinstance(gps_info, dict):
-            try:
-                def _to_deg(vals):
-                    d, m, s = [float(v) for v in vals]
-                    return d + m / 60 + s / 3600
-                if 2 in gps_info and 4 in gps_info:
-                    lat = _to_deg(gps_info[2])
-                    lng = _to_deg(gps_info[4])
-                    if gps_info.get(1) == 'S': lat = -lat
-                    if gps_info.get(3) == 'W': lng = -lng
-                    result["gps_lat"] = f"{lat:.6f}"
-                    result["gps_lng"] = f"{lng:.6f}"
-            except Exception:
-                pass
-    except Exception as e:
-        # User-visible failure (photo loses metadata): surface at WARNING
-        # and record on the result so the upload endpoint can pass it back.
-        logger.warning(f"EXIF extraction failed: {e}")
-        result["exif_error"] = str(e)
-    return result
-
-
-# ---- Helpers ----
-
-def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any]:
-    return {
-        "id": img.id,
-        "filename": img.filename,
-        "url": f"/api/generated-image/{img.filename}",
-        "prompt": img.prompt,
-        "model": img.model,
-        "size": img.size,
-        "quality": img.quality,
-        "tags": img.tags or "",
-        "ai_tags": img.ai_tags or "",
-        "user_tags": img.tags or "",
-        "session_id": img.session_id,
-        "session_name": session_name,
-        "album_id": img.album_id,
-        "is_active": img.is_active,
-        "favorite": img.favorite or False,
-        "taken_at": img.taken_at.isoformat() if img.taken_at else None,
-        "camera": f"{img.camera_make or ''} {img.camera_model or ''}".strip() or None,
-        "gps": {"lat": img.gps_lat, "lng": img.gps_lng} if img.gps_lat else None,
-        "width": img.width,
-        "height": img.height,
-        "file_size": img.file_size,
-        "created_at": img.created_at.isoformat() if img.created_at else None,
-        "updated_at": img.updated_at.isoformat() if img.updated_at else None,
-    }
-
-
-def _owner_filter(q, user, model_cls=GalleryImage):
-    """Apply owner filtering to a gallery query.
-
-    ``get_current_user`` returns None both in auth-disabled single-user mode
-    and when auth is enabled but no current user was resolved. Preserve the
-    single-user behavior, but fail closed for auth-enabled null-user states.
-    """
-    if user is not None:
-        return q.filter(model_cls.owner == user)
-    if _auth_disabled():
-        return q
-    return q.filter(False)
-
-
-
-def _human_size(nbytes):
-    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
-        if abs(nbytes) < 1024:
-            return f"{nbytes:.1f} {unit}"
-        nbytes /= 1024
-    return f"{nbytes:.1f} PB"
+_sys.modules[__name__] = _canonical

routes/history_routes.py

@@ -3,7 +3,8 @@
 import json
 import uuid
 import logging
-from typing import Dict, Any
+import re
+from typing import Dict, Any, Optional
 
 from fastapi import APIRouter, Request, HTTPException
 
@@ -19,6 +20,63 @@ from routes.session_routes import (
 
 logger = logging.getLogger(__name__)
 
+_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
+_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
+
+
+def _history_display_content(content: Any) -> Any:
+    """Return a lightweight browser-display copy of stored message content.
+
+    Older multimodal user messages may be persisted as a JSON *string*
+    containing image_url blocks with inline base64 image bytes. Those bytes are
+    needed for model calls when the turn is first sent, but they should not be
+    sent back through /api/history every time the user opens the chat. The
+    attachment metadata already carries file ids/names for the UI cards.
+    """
+    if isinstance(content, list):
+        text_parts = []
+        omitted_media = 0
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "text":
+                text = block.get("text")
+                if isinstance(text, str) and text:
+                    text_parts.append(text)
+            elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
+                omitted_media += 1
+        text = "\n".join(text_parts).strip()
+        if omitted_media and not text:
+            return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
+        return text
+
+    if not isinstance(content, str):
+        return content
+    if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
+        return content
+
+    stripped = content.lstrip()
+    if stripped.startswith("["):
+        try:
+            blocks = json.loads(content)
+        except (json.JSONDecodeError, TypeError, ValueError):
+            blocks = None
+        if isinstance(blocks, list):
+            text_parts = []
+            for block in blocks:
+                if not isinstance(block, dict):
+                    continue
+                if block.get("type") == "text":
+                    text = block.get("text")
+                    if isinstance(text, str) and text:
+                        text_parts.append(text)
+            if text_parts:
+                return "\n".join(text_parts).strip()
+
+    if "data:image/" in content:
+        return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
+    return content
+
 
 def _merge_continue_rows_to_delete(db_messages, db1, db2):
     """DB rows to delete when merging the last two assistant messages.
@@ -43,9 +101,69 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2):
 def setup_history_routes(session_manager) -> APIRouter:
     router = APIRouter(tags=["history"])
 
+    def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
+        entry = {"role": m.role, "content": _history_display_content(m.content)}
+        meta = {}
+        if m.meta_data:
+            try:
+                meta = json.loads(m.meta_data) or {}
+            except (json.JSONDecodeError, ValueError):
+                meta = {}
+        if m.timestamp and "timestamp" not in meta:
+            meta["timestamp"] = m.timestamp.isoformat() + "Z"
+        if meta:
+            entry["metadata"] = meta
+        return entry
+
     @router.get("/api/history/{session_id}")
-    async def get_session_history(request: Request, session_id: str) -> Dict[str, Any]:
+    async def get_session_history(
+        request: Request,
+        session_id: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Dict[str, Any]:
         _verify_session_owner(request, session_id)
+        if limit is not None:
+            page_limit = max(1, min(int(limit), 100))
+            db = SessionLocal()
+            try:
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session is None:
+                    raise HTTPException(404, f"Session '{session_id}' not found")
+
+                total = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .count()
+                )
+                page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
+                page_offset = max(0, min(page_offset, total))
+                rows = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .offset(page_offset)
+                    .limit(page_limit)
+                    .all()
+                )
+                history_dict = [
+                    entry for entry in (_db_history_entry(m) for m in rows)
+                    if not (entry.get("metadata") or {}).get("hidden")
+                ]
+                return {
+                    "history": history_dict,
+                    "model": db_session.model,
+                    "endpoint_url": db_session.endpoint_url,
+                    "name": db_session.name,
+                    "offset": page_offset,
+                    "limit": page_limit,
+                    "total": total,
+                    "has_more_before": page_offset > 0,
+                    "has_more_after": page_offset + len(rows) < total,
+                }
+            finally:
+                db.close()
+
         try:
             session = session_manager.get_session(session_id)
         except KeyError:
@@ -57,7 +175,7 @@ def setup_history_routes(session_manager) -> APIRouter:
                 # Skip hidden messages (e.g. compaction summaries for AI context)
                 if msg.metadata and msg.metadata.get("hidden"):
                     continue
-                entry = {"role": msg.role, "content": msg.content}
+                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
                 if msg.metadata:
                     entry["metadata"] = msg.metadata
                 history_dict.append(entry)
@@ -66,7 +184,7 @@ def setup_history_routes(session_manager) -> APIRouter:
                     continue
                 entry = {
                     "role": msg.get("role", ""),
-                    "content": msg.get("content", ""),
+                    "content": _history_display_content(msg.get("content", "")),
                 }
                 if msg.get("metadata"):
                     entry["metadata"] = msg["metadata"]
@@ -82,21 +200,9 @@ def setup_history_routes(session_manager) -> APIRouter:
                     .order_by(DbChatMessage.timestamp)
                     .all()
                 )
-                import json as _json
                 db_history = []
                 for m in db_messages:
-                    entry = {"role": m.role, "content": m.content}
-                    meta = {}
-                    if m.meta_data:
-                        try:
-                            meta = _json.loads(m.meta_data) or {}
-                        except (json.JSONDecodeError, ValueError):
-                            meta = {}
-                    if m.timestamp and "timestamp" not in meta:
-                        meta["timestamp"] = m.timestamp.isoformat() + "Z"
-                    if meta:
-                        entry["metadata"] = meta
-                    db_history.append(entry)
+                    db_history.append(_db_history_entry(m))
                 if db_history:
                     # Rebuild in-memory history from the full set so hidden
                     # messages (e.g. compaction summaries) are kept for AI context.

routes/hwfit_routes.py

@@ -1,8 +1,13 @@
+import json
+import os
 import re
+import shlex
+import subprocess
 from copy import deepcopy
 
 from fastapi import APIRouter, HTTPException
 
+from core.platform_compat import run_ssh_command
 from routes._validators import validate_remote_host, validate_ssh_port
 
 
@@ -107,6 +112,73 @@ def _apply_manual_hardware(system, manual_mode="", manual_gpu_count="", manual_v
     return system
 
 
+def _run_model_probe(host: str, ssh_port: str, cmd: str) -> str:
+    try:
+        if host:
+            r = run_ssh_command(
+                host,
+                ssh_port or None,
+                cmd,
+                timeout=15,
+                connect_timeout=5,
+                strict_host_key_checking=False,
+                text=True,
+            )
+        else:
+            r = subprocess.run(["bash", "-lc", cmd], capture_output=True, text=True, timeout=15)
+        if r.returncode == 0:
+            return (r.stdout or "").strip()
+    except Exception:
+        return ""
+    return ""
+
+
+def _inspect_model_path(model_path: str, host: str = "", ssh_port: str = "") -> dict:
+    """Read lightweight metadata from a local or SSH-visible HF model folder."""
+    path = (model_path or "").strip()
+    if not path or path.startswith(("http://", "https://")):
+        return {}
+    if not (path.startswith("/") or path.startswith("~")):
+        return {}
+
+    qpath = shlex.quote(path)
+    qconfig = shlex.quote(os.path.join(path, "config.json"))
+    out = {}
+    exists = _run_model_probe(host, ssh_port, f"test -d {qpath} && printf found || printf missing")
+    if exists != "found":
+        target = host or "local container"
+        out["model_probe_error"] = f"Model path is not visible on {target}: {path}"
+        return out
+    raw_config = _run_model_probe(host, ssh_port, f"test -f {qconfig} && sed -n '1,240p' {qconfig}")
+    if raw_config:
+        try:
+            cfg = json.loads(raw_config)
+        except Exception:
+            cfg = {}
+        for key in ("context_length", "max_position_embeddings", "n_ctx_train", "model_max_length", "max_seq_len"):
+            value = cfg.get(key)
+            if isinstance(value, (int, float)) and value > 0:
+                out["model_ctx_max"] = int(value)
+                break
+    else:
+        out["model_probe_error"] = f"config.json not found in model path: {path}"
+
+    size_cmd = (
+        f"find {qpath} -type f \\( -name '*.safetensors' -o -name '*.bin' -o -name '*.gguf' \\) "
+        "-printf '%s\\n' 2>/dev/null | awk '{s+=$1} END {if (s>0) printf \"%.6f\", s/1073741824}'"
+    )
+    weights = _run_model_probe(host, ssh_port, size_cmd)
+    try:
+        weights_gb = float(weights)
+    except Exception:
+        weights_gb = 0.0
+    if weights_gb > 0:
+        out["model_weights_gb"] = round(weights_gb, 3)
+    elif "model_probe_error" not in out:
+        out["model_probe_error"] = f"No model weight files found in: {path}"
+    return out
+
+
 def setup_hwfit_routes():
     router = APIRouter(prefix="/api/hwfit", tags=["hwfit"])
 
@@ -235,7 +307,7 @@ def setup_hwfit_routes():
         return {"system": system, "models": results}
 
     @router.get("/profiles")
-    def get_serve_profiles(model: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):
+    def get_serve_profiles(model: str = "", model_path: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):
         """Compute llama.cpp serve profiles (Quality/Balanced/Speed) for `model`
         against the detected hardware on `host` (or local). Returns concrete
         flags (n_gpu_layers, n_cpu_moe, cache_type, ctx) the serve UI can apply.
@@ -260,8 +332,23 @@ def setup_hwfit_routes():
             # "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct".
             s = (s or "").lower().strip()
             s = s.split("/")[-1]                     # drop org prefix
-            s = re.sub(r"[-_.]?gguf$", "", s)        # drop trailing gguf marker
-            s = re.sub(r"[-_.](q\d[^/]*|iq\d[^/]*|fp8|bf16|f16|awq[^/]*|gptq[^/]*)$", "", s)
+            for suffix in ("-gguf", "_gguf", ".gguf", "gguf"):
+                if s.endswith(suffix):
+                    s = s[: -len(suffix)]
+                    break
+            cut_at = None
+            for idx, ch in enumerate(s):
+                if ch not in "-_." or idx + 1 >= len(s):
+                    continue
+                suffix = s[idx + 1:]
+                if (
+                    suffix in {"fp8", "bf16", "f16"}
+                    or suffix.startswith(("awq", "gptq", "iq"))
+                    or (suffix.startswith("q") and len(suffix) > 1 and suffix[1].isdigit())
+                ):
+                    cut_at = idx
+            if cut_at is not None:
+                s = s[:cut_at]
             return s
 
         m = catalog.get(model)
@@ -272,8 +359,16 @@ def setup_hwfit_routes():
                 if nn and (nn == want or want.endswith(nn) or nn.endswith(want)):
                     m = entry
                     break
+        path_meta = _inspect_model_path(model_path or model, host=host, ssh_port=ssh_port)
         if m is None:
-            return {"system": system, "profiles": [], "error": "model not in catalog"}
+            return {
+                "system": system,
+                "profiles": [],
+                "error": "model not in catalog",
+                "model_ctx_max": int(path_meta.get("model_ctx_max") or 0),
+                "model_weights_gb": float(path_meta.get("model_weights_gb") or 0),
+                "model_probe_error": path_meta.get("model_probe_error") or "",
+            }
         # Surface the model's trained context limit so the serve UI can clamp a
         # user-typed context down to it (asking for ctx > n_ctx_train overflows
         # and, with a quantized KV cache, can crash the GPU).
@@ -283,6 +378,16 @@ def setup_hwfit_routes():
             if isinstance(v, (int, float)) and v > 0:
                 model_ctx_max = int(v)
                 break
+        path_ctx_max = int(path_meta.get("model_ctx_max") or 0)
+        if path_ctx_max > 0:
+            model_ctx_max = max(model_ctx_max, path_ctx_max)
+        model_weights_gb = float(path_meta.get("model_weights_gb") or 0)
+        if model_weights_gb <= 0:
+            for k in ("min_vram_gb", "required_gb", "size_gb", "recommended_ram_gb", "min_ram_gb"):
+                v = m.get(k)
+                if isinstance(v, (int, float)) and v > 0:
+                    model_weights_gb = float(v)
+                    break
         return {
             "system": system,
             "profiles": compute_serve_profiles(
@@ -291,6 +396,8 @@ def setup_hwfit_routes():
                 serve_quant=(serve_quant or None),
             ),
             "model_ctx_max": model_ctx_max,
+            "model_weights_gb": model_weights_gb,
+            "model_probe_error": path_meta.get("model_probe_error") or "",
         }
 
     @router.get("/image-models")

routes/memory/__init__.py

@@ -0,0 +1,5 @@
+"""Memory route domain package (slice 2c, #4082/#4071).
+
+Contains memory_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/memory_routes.py re-exports from here.
+"""

routes/memory/memory_routes.py

@@ -0,0 +1,552 @@
+# routes/memory_routes.py
+from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
+from typing import Dict, Any, Optional, List
+import json
+import os
+import re
+import tempfile
+import time
+from datetime import datetime
+import logging
+
+# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
+# Strips one prefix per call so import-from-LLM-output doesn't leave the
+# numbering inside the saved memory text. Bullet markers (-, *, •) are
+# also peeled here for the same reason.
+_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
+
+
+def _strip_list_prefix(text: str) -> str:
+    if not text:
+        return text
+    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
+
+from services.memory import MemoryManager
+from core.session_manager import SessionManager
+from src.request_models import MemoryAddRequest
+from core.database import SessionLocal
+from src.llm_core import llm_call_async
+from services.memory.memory_extractor import audit_memories
+from src.auth_helpers import get_current_user, require_user
+from src.endpoint_resolver import resolve_endpoint
+from src.task_endpoint import resolve_task_endpoint
+from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
+
+logger = logging.getLogger(__name__)
+
+
+def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
+    """Set up memory-related routes."""
+    router = APIRouter(prefix="/api/memory", tags=["memory"])
+
+    def _owner(request: Request) -> Optional[str]:
+        return get_current_user(request)
+
+    def _assert_session_owner(session_obj, user):
+        """SECURITY: 404 if the caller does not own this session.
+
+        SessionManager.get_session is NOT owner-scoped — it returns any
+        session by id. These routes accept a caller-supplied session id, so
+        without this gate a user could target another tenant's session and
+        leak their chat history, their session-scoped LLM credentials, or the
+        session title. Mirrors session_routes / webhook_routes ownership.
+        """
+        if user is not None and getattr(session_obj, "owner", None) != user:
+            raise HTTPException(404, "Session not found")
+
+    def _verify_memory_owner(memory: dict, user: Optional[str]):
+        """Raise 404 if user doesn't own this memory.
+
+        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
+        allowed any user to read/edit/delete memories with an empty/null owner
+        field, which leaked legacy data across the multi-user deploy.
+        """
+        if user is None:
+            return  # Auth disabled
+        if memory.get("owner") != user:
+            raise HTTPException(404, "Memory not found")
+
+    @router.post("/debug")
+    def debug_memory_relevance(request: Request, query: str = Form(...)):
+        """Debug which memories would be triggered for a query"""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
+
+        return {
+            "query": query,
+            "total_memories": len(memories),
+            "relevant_count": len(relevant),
+            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
+                                 for m in relevant]
+        }
+
+    @router.post("/add", response_model=Dict[str, Any])
+    async def api_add_memory(
+        request: Request,
+        memory_data: Optional[MemoryAddRequest] = None
+    ):
+        """Add a new memory entry with optional category, source, and session reference."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+        if memory_data is None:
+            form = await request.form()
+            memory_data = MemoryAddRequest(
+                text=form.get("text"),
+                category=form.get("category", "fact"),
+                source=form.get("source", "user"),
+                session_id=form.get("session_id")
+            )
+
+        user = _owner(request)
+        text = (memory_data.text or "").strip()
+        if not text:
+            raise HTTPException(400, "empty memory")
+        user_mem = memory_manager.load(owner=user)
+        if memory_manager.find_duplicates(text, user_mem):
+            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
+
+        if memory_data.session_id:
+            try:
+                session_obj = session_manager.get_session(memory_data.session_id)
+            except KeyError:
+                raise HTTPException(404, "Session not found")
+            _assert_session_owner(session_obj, user)
+
+        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
+        if memory_data.session_id:
+            new_entry["session_id"] = memory_data.session_id
+        all_mem = memory_manager.load_all()
+        all_mem.append(new_entry)
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.add(new_entry["id"], text)
+        try:
+            from src.event_bus import fire_event
+            fire_event("memory_added", user)
+        except Exception:
+            logger.debug("memory_added event dispatch failed", exc_info=True)
+        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
+
+    @router.get("")
+    def api_get_memory(request: Request):
+        """Return all memory entries with their metadata."""
+        user = _owner(request)
+        return {"memory": memory_manager.load(owner=user)}
+
+    @router.post("/search")
+    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
+        """Search across all memories with optional filters."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+
+        if session_id:
+            memories = [m for m in memories if m.get("session_id") == session_id]
+
+        if category:
+            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
+
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+
+        return {"memories": relevant, "total": len(relevant), "query": query}
+
+    @router.get("/timeline")
+    def memory_timeline(request: Request):
+        """Get memories in chronological order with source session information."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        results = []
+        for memory in sorted_memories:
+            if "timestamp" in memory:
+                try:
+                    dt = datetime.fromtimestamp(memory["timestamp"])
+                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
+                except (ValueError, OSError, OverflowError):
+                    memory["timestamp_str"] = "Unknown"
+            else:
+                memory["timestamp_str"] = "Unknown"
+
+            session_id = memory.get("session_id")
+            if session_id and session_id in session_manager.sessions:
+                try:
+                    session = session_manager.get_session(session_id)
+                    if session:
+                        _assert_session_owner(session, user)
+                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
+                except KeyError:
+                    memory["session_name"] = "Unknown"
+                except HTTPException as exc:
+                    if exc.status_code != 404:
+                        raise
+                    memory["session_name"] = "Unknown"
+            else:
+                memory["session_name"] = "Unknown"
+
+            results.append(memory)
+
+        return {"timeline": results, "total": len(results)}
+
+    @router.get("/by-session/{session_id}")
+    def get_memory_by_session(request: Request, session_id: str):
+        """Get all memories associated with a specific session."""
+        user = _owner(request)
+        try:
+            _session_obj = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, f"Session {session_id} not found")
+        _assert_session_owner(_session_obj, user)
+        memories = memory_manager.load(owner=user)
+        session_memories = [m for m in memories if m.get("session_id") == session_id]
+
+        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        try:
+            session = session_manager.get_session(session_id)
+            session_name = session.name if session else f"Session {session_id[:6]}"
+        except KeyError:
+            session_name = f"Session {session_id[:6]}"
+
+        for memory in session_memories:
+            memory["session_name"] = session_name
+
+        return {
+            "session_id": session_id,
+            "session_name": session_name,
+            "memory_count": len(session_memories),
+            "memories": session_memories
+        }
+
+    @router.post("/extract")
+    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
+        """Analyze a session's chat history and return memory suggestions."""
+        require_user(request)
+        try:
+            sess = session_manager.get_session(session)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        _assert_session_owner(sess, _owner(request))
+
+        system_msg = {
+            "role": "system",
+            "content": (
+                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
+                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
+                "might want to remember for future interactions. Return each piece of information as a JSON object "
+                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
+                "Only include information that is specific and likely to be useful later."
+            ),
+        }
+        messages = [system_msg] + sess.get_context_messages()
+
+        t_url, t_model, t_headers = resolve_task_endpoint(
+            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
+        )
+
+        try:
+            suggestion_text = await llm_call_async(
+                t_url,
+                t_model,
+                messages,
+                temperature=0.2,
+                max_tokens=500,
+                headers=t_headers,
+            )
+            try:
+                suggestions = json.loads(suggestion_text)
+                if isinstance(suggestions, list):
+                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
+                else:
+                    suggestions = []
+            except json.JSONDecodeError:
+                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
+
+            return {"suggestions": [s for s in suggestions if s]}
+        except Exception as e:
+            logger.error(f"LLM memory extraction failed (session {session}): {e}")
+            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
+            return {"suggestions": [item["text"] for item in fallback]}
+
+    @router.post("/audit")
+    async def api_audit_memories(request: Request, session: str = Form(None)):
+        """Deduplicate and consolidate memories via LLM.
+
+        Uses task/utility/default settings through the shared resolver, with
+        the active session as fallback when no task or utility model is set.
+        Returns before and after memory counts.
+        """
+        user = _owner(request)
+        fallback_url = fallback_model = None
+        fallback_headers = None
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+                fallback_url = sess.endpoint_url
+                fallback_model = sess.model
+                fallback_headers = sess.headers
+            except KeyError:
+                pass
+
+        endpoint_url, model, headers = resolve_task_endpoint(
+            fallback_url, fallback_model, fallback_headers, owner=user
+        )
+
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No default model configured — set one in Settings")
+
+        result = await audit_memories(
+            memory_manager,
+            memory_vector,
+            endpoint_url,
+            model,
+            headers,
+            owner=user,
+        )
+
+        if "error" in result and "before" not in result:
+            raise HTTPException(502, f"Audit failed: {result['error']}")
+
+        return {
+            "ok": "error" not in result,
+            "before": result.get("before", 0),
+            "after": result.get("after", 0),
+            "removed": result.get("before", 0) - result.get("after", 0),
+            # True when the audit skipped the LLM because nothing changed
+            # since the last tidy. Frontend already says "Already clean"
+            # for removed==0, so this is here for future use / debugging.
+            "already_tidy": bool(result.get("already_tidy")),
+        }
+
+    @router.post("/import")
+    async def import_memories_from_file(
+        request: Request,
+        session: str | None = Form(None),
+        file: UploadFile = File(...)
+    ):
+        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+
+        endpoint_url = None
+        model = None
+        headers = {}
+
+        user = _owner(request)
+
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+            except KeyError:
+                sess = None
+            except HTTPException as exc:
+                if exc.status_code != 404:
+                    raise
+                sess = None
+
+            if sess is None:
+                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
+                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
+            else:
+                endpoint_url, model, headers = resolve_task_endpoint(
+                    sess.endpoint_url, sess.model, sess.headers, owner=user
+                )
+        else:
+            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
+    
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
+
+        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
+        filename = file.filename or "upload"
+        _, ext = os.path.splitext(filename.lower())
+
+        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
+        if ext not in allowed:
+            raise HTTPException(400, f"Unsupported file type: {ext}")
+
+        # Extract text based on file type
+        if ext == ".pdf":
+            from src.document_processor import _process_pdf
+            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+                tmp.write(content)
+                tmp_path = tmp.name
+            try:
+                text = _process_pdf(tmp_path, owner=_owner(request))
+            finally:
+                os.unlink(tmp_path)
+        else:
+            try:
+                text = content.decode("utf-8")
+            except UnicodeDecodeError:
+                from charset_normalizer import detect
+                encoding = (detect(content) or {}).get("encoding") or "utf-8"
+                text = content.decode(encoding, errors="replace")
+
+        if not text.strip():
+            return {"suggestions": [], "message": "No readable content found"}
+
+        # Fast path: a .json upload that already looks like a memories export
+        # (list of {text, category, ...} dicts, or list of strings) round-trips
+        # directly without spending an LLM call to re-extract its own output.
+        # Without this, re-importing a memories.json from another account
+        # ran the file through the extractor, which often re-emitted the
+        # entries as a numbered list (and the numbering leaked into the
+        # `text` field).
+        if ext == ".json":
+            try:
+                parsed = json.loads(text)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list) and parsed:
+                direct = []
+                for item in parsed:
+                    if isinstance(item, dict) and item.get("text"):
+                        direct.append({
+                            "text": _strip_list_prefix(str(item["text"])),
+                            "category": item.get("category") or "fact",
+                        })
+                    elif isinstance(item, str) and item.strip():
+                        direct.append({
+                            "text": _strip_list_prefix(item.strip()),
+                            "category": "fact",
+                        })
+                if direct:
+                    return {"suggestions": direct, "filename": filename}
+
+        # Truncate very long documents
+        if len(text) > 15000:
+            text = text[:15000] + "\n[Truncated]"
+
+        # Send to LLM for memory extraction
+        import_prompt = (
+            "You are a memory extraction assistant. The user uploaded a document. "
+            "Analyze the text below and extract specific, useful facts — things like "
+            "names, preferences, jobs, locations, relationships, opinions, projects, "
+            "goals, contacts, or any other personal details worth remembering.\n\n"
+            "Rules:\n"
+            "- Each fact should be a short, self-contained statement\n"
+            "- Do NOT extract generic knowledge\n"
+            "- Focus on personal, memorable information\n"
+            "- If there are no useful facts, return an empty array\n\n"
+            "Return a JSON array of objects with 'text' and 'category' fields.\n"
+            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
+            "Return ONLY valid JSON, no markdown fences."
+        )
+
+        try:
+            raw = await llm_call_async(
+                endpoint_url,
+                model,
+                [
+                    {"role": "system", "content": import_prompt},
+                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
+                ],
+                temperature=0.2,
+                max_tokens=2000,
+                headers=headers,
+            )
+
+            # Parse JSON
+            raw = raw.strip()
+            if raw.startswith("```"):
+                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+
+            suggestions = json.loads(raw)
+            if isinstance(suggestions, list):
+                normalized = []
+                for s in suggestions:
+                    if not s:
+                        continue
+                    if isinstance(s, dict):
+                        s = dict(s)
+                        if s.get("text"):
+                            s["text"] = _strip_list_prefix(str(s["text"]))
+                        normalized.append(s)
+                    else:
+                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
+                suggestions = normalized
+            else:
+                suggestions = []
+
+            return {"suggestions": suggestions, "filename": filename}
+
+        except json.JSONDecodeError:
+            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
+            # numbering the model added so saved memories don't keep the prefix.
+            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
+            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
+        except Exception as e:
+            logger.error(f"Memory import extraction failed: {e}")
+            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
+
+    @router.post("/{memory_id}/pin")
+    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
+        """Pin or unpin a memory. Pinned memories are always included in context."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["pinned"] = pinned
+                memory_manager.save(all_mem)
+                return {"ok": True, "pinned": pinned}
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
+    @router.get("/{memory_id}")
+    def get_memory_item(request: Request, memory_id: str):
+        """Get a specific memory item by ID."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        for memory in memories:
+            if memory["id"] == memory_id:
+                return {"memory": memory}
+
+        raise HTTPException(404, "Memory not found")
+
+    @router.put("/{memory_id}")
+    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
+        """Update an existing memory item with new text and optional category."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["text"] = text.strip()
+                if category:
+                    all_mem[i]["category"] = category
+                all_mem[i]["timestamp"] = int(time.time())
+
+                memory_manager.save(all_mem)
+                # Sync vector index (remove old, add updated)
+                if memory_vector and memory_vector.healthy:
+                    memory_vector.remove(memory_id)
+                    memory_vector.add(memory_id, text.strip())
+                return {"ok": True, "message": "Memory updated successfully"}
+
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    @router.delete("/{memory_id}")
+    def delete_memory(request: Request, memory_id: str):
+        """Delete a memory item by its ID."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+
+        # Find and verify ownership before deleting
+        target = next((m for m in all_mem if m["id"] == memory_id), None)
+        if not target:
+            raise HTTPException(404, f"Memory item {memory_id} not found")
+        _verify_memory_owner(target, user)
+
+        all_mem = [m for m in all_mem if m["id"] != memory_id]
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.remove(memory_id)
+        return {"ok": True, "message": "Memory deleted successfully"}
+
+    return router

routes/memory_routes.py

@@ -1,552 +1,18 @@
-# routes/memory_routes.py
-from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
-from typing import Dict, Any, Optional, List
-import json
-import os
-import re
-import tempfile
-import time
-from datetime import datetime
-import logging
+"""Backward-compat shim — canonical location is routes/memory/memory_routes.py.
 
-# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
-# Strips one prefix per call so import-from-LLM-output doesn't leave the
-# numbering inside the saved memory text. Bullet markers (-, *, •) are
-# also peeled here for the same reason.
-_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.memory_routes``, ``from routes.memory_routes import X``,
+``importlib.import_module("routes.memory_routes")``, and
+``monkeypatch.setattr(routes.memory_routes, "ATTR", ...)`` (used by
+test_memory_routes_session_owner.py and test_memory_owner_isolation.py via
+``import ... as mr`` + ``setattr(mr, ...)``) all operate on the *same* object
+the application actually uses. Keeps existing import paths working after
+slice 2c (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
+import sys as _sys
 
-def _strip_list_prefix(text: str) -> str:
-    if not text:
-        return text
-    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
+from routes.memory import memory_routes as _canonical  # noqa: F401
 
-from services.memory import MemoryManager
-from core.session_manager import SessionManager
-from src.request_models import MemoryAddRequest
-from core.database import SessionLocal
-from src.llm_core import llm_call_async
-from services.memory.memory_extractor import audit_memories
-from src.auth_helpers import get_current_user, require_user
-from src.endpoint_resolver import resolve_endpoint
-from src.task_endpoint import resolve_task_endpoint
-from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
-
-logger = logging.getLogger(__name__)
-
-
-def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
-    """Set up memory-related routes."""
-    router = APIRouter(prefix="/api/memory", tags=["memory"])
-
-    def _owner(request: Request) -> Optional[str]:
-        return get_current_user(request)
-
-    def _assert_session_owner(session_obj, user):
-        """SECURITY: 404 if the caller does not own this session.
-
-        SessionManager.get_session is NOT owner-scoped — it returns any
-        session by id. These routes accept a caller-supplied session id, so
-        without this gate a user could target another tenant's session and
-        leak their chat history, their session-scoped LLM credentials, or the
-        session title. Mirrors session_routes / webhook_routes ownership.
-        """
-        if user is not None and getattr(session_obj, "owner", None) != user:
-            raise HTTPException(404, "Session not found")
-
-    def _verify_memory_owner(memory: dict, user: Optional[str]):
-        """Raise 404 if user doesn't own this memory.
-
-        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
-        allowed any user to read/edit/delete memories with an empty/null owner
-        field, which leaked legacy data across the multi-user deploy.
-        """
-        if user is None:
-            return  # Auth disabled
-        if memory.get("owner") != user:
-            raise HTTPException(404, "Memory not found")
-
-    @router.post("/debug")
-    def debug_memory_relevance(request: Request, query: str = Form(...)):
-        """Debug which memories would be triggered for a query"""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
-
-        return {
-            "query": query,
-            "total_memories": len(memories),
-            "relevant_count": len(relevant),
-            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
-                                 for m in relevant]
-        }
-
-    @router.post("/add", response_model=Dict[str, Any])
-    async def api_add_memory(
-        request: Request,
-        memory_data: Optional[MemoryAddRequest] = None
-    ):
-        """Add a new memory entry with optional category, source, and session reference."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-        if memory_data is None:
-            form = await request.form()
-            memory_data = MemoryAddRequest(
-                text=form.get("text"),
-                category=form.get("category", "fact"),
-                source=form.get("source", "user"),
-                session_id=form.get("session_id")
-            )
-
-        user = _owner(request)
-        text = (memory_data.text or "").strip()
-        if not text:
-            raise HTTPException(400, "empty memory")
-        user_mem = memory_manager.load(owner=user)
-        if memory_manager.find_duplicates(text, user_mem):
-            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
-
-        if memory_data.session_id:
-            try:
-                session_obj = session_manager.get_session(memory_data.session_id)
-            except KeyError:
-                raise HTTPException(404, "Session not found")
-            _assert_session_owner(session_obj, user)
-
-        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
-        if memory_data.session_id:
-            new_entry["session_id"] = memory_data.session_id
-        all_mem = memory_manager.load_all()
-        all_mem.append(new_entry)
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.add(new_entry["id"], text)
-        try:
-            from src.event_bus import fire_event
-            fire_event("memory_added", user)
-        except Exception:
-            logger.debug("memory_added event dispatch failed", exc_info=True)
-        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
-
-    @router.get("")
-    def api_get_memory(request: Request):
-        """Return all memory entries with their metadata."""
-        user = _owner(request)
-        return {"memory": memory_manager.load(owner=user)}
-
-    @router.post("/search")
-    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
-        """Search across all memories with optional filters."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-
-        if session_id:
-            memories = [m for m in memories if m.get("session_id") == session_id]
-
-        if category:
-            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
-
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
-
-        return {"memories": relevant, "total": len(relevant), "query": query}
-
-    @router.get("/timeline")
-    def memory_timeline(request: Request):
-        """Get memories in chronological order with source session information."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        results = []
-        for memory in sorted_memories:
-            if "timestamp" in memory:
-                try:
-                    dt = datetime.fromtimestamp(memory["timestamp"])
-                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
-                except (ValueError, OSError, OverflowError):
-                    memory["timestamp_str"] = "Unknown"
-            else:
-                memory["timestamp_str"] = "Unknown"
-
-            session_id = memory.get("session_id")
-            if session_id and session_id in session_manager.sessions:
-                try:
-                    session = session_manager.get_session(session_id)
-                    if session:
-                        _assert_session_owner(session, user)
-                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
-                except KeyError:
-                    memory["session_name"] = "Unknown"
-                except HTTPException as exc:
-                    if exc.status_code != 404:
-                        raise
-                    memory["session_name"] = "Unknown"
-            else:
-                memory["session_name"] = "Unknown"
-
-            results.append(memory)
-
-        return {"timeline": results, "total": len(results)}
-
-    @router.get("/by-session/{session_id}")
-    def get_memory_by_session(request: Request, session_id: str):
-        """Get all memories associated with a specific session."""
-        user = _owner(request)
-        try:
-            _session_obj = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session {session_id} not found")
-        _assert_session_owner(_session_obj, user)
-        memories = memory_manager.load(owner=user)
-        session_memories = [m for m in memories if m.get("session_id") == session_id]
-
-        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        try:
-            session = session_manager.get_session(session_id)
-            session_name = session.name if session else f"Session {session_id[:6]}"
-        except KeyError:
-            session_name = f"Session {session_id[:6]}"
-
-        for memory in session_memories:
-            memory["session_name"] = session_name
-
-        return {
-            "session_id": session_id,
-            "session_name": session_name,
-            "memory_count": len(session_memories),
-            "memories": session_memories
-        }
-
-    @router.post("/extract")
-    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
-        """Analyze a session's chat history and return memory suggestions."""
-        require_user(request)
-        try:
-            sess = session_manager.get_session(session)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        _assert_session_owner(sess, _owner(request))
-
-        system_msg = {
-            "role": "system",
-            "content": (
-                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
-                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
-                "might want to remember for future interactions. Return each piece of information as a JSON object "
-                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
-                "Only include information that is specific and likely to be useful later."
-            ),
-        }
-        messages = [system_msg] + sess.get_context_messages()
-
-        t_url, t_model, t_headers = resolve_task_endpoint(
-            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
-        )
-
-        try:
-            suggestion_text = await llm_call_async(
-                t_url,
-                t_model,
-                messages,
-                temperature=0.2,
-                max_tokens=500,
-                headers=t_headers,
-            )
-            try:
-                suggestions = json.loads(suggestion_text)
-                if isinstance(suggestions, list):
-                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
-                else:
-                    suggestions = []
-            except json.JSONDecodeError:
-                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
-
-            return {"suggestions": [s for s in suggestions if s]}
-        except Exception as e:
-            logger.error(f"LLM memory extraction failed (session {session}): {e}")
-            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
-            return {"suggestions": [item["text"] for item in fallback]}
-
-    @router.post("/audit")
-    async def api_audit_memories(request: Request, session: str = Form(None)):
-        """Deduplicate and consolidate memories via LLM.
-
-        Uses task/utility/default settings through the shared resolver, with
-        the active session as fallback when no task or utility model is set.
-        Returns before and after memory counts.
-        """
-        user = _owner(request)
-        fallback_url = fallback_model = None
-        fallback_headers = None
-        if session:
-            try:
-                sess = session_manager.get_session(session)
-                _assert_session_owner(sess, user)
-                fallback_url = sess.endpoint_url
-                fallback_model = sess.model
-                fallback_headers = sess.headers
-            except KeyError:
-                pass
-
-        endpoint_url, model, headers = resolve_task_endpoint(
-            fallback_url, fallback_model, fallback_headers, owner=user
-        )
-
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No default model configured — set one in Settings")
-
-        result = await audit_memories(
-            memory_manager,
-            memory_vector,
-            endpoint_url,
-            model,
-            headers,
-            owner=user,
-        )
-
-        if "error" in result and "before" not in result:
-            raise HTTPException(502, f"Audit failed: {result['error']}")
-
-        return {
-            "ok": "error" not in result,
-            "before": result.get("before", 0),
-            "after": result.get("after", 0),
-            "removed": result.get("before", 0) - result.get("after", 0),
-            # True when the audit skipped the LLM because nothing changed
-            # since the last tidy. Frontend already says "Already clean"
-            # for removed==0, so this is here for future use / debugging.
-            "already_tidy": bool(result.get("already_tidy")),
-        }
-
-    @router.post("/import")
-    async def import_memories_from_file(
-        request: Request,
-        session: str | None = Form(None),
-        file: UploadFile = File(...)
-    ):
-        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-
-        endpoint_url = None
-        model = None
-        headers = {}
-
-        user = _owner(request)
-
-        if session:
-            try:
-                sess = session_manager.get_session(session)
-                _assert_session_owner(sess, user)
-            except KeyError:
-                sess = None
-            except HTTPException as exc:
-                if exc.status_code != 404:
-                    raise
-                sess = None
-
-            if sess is None:
-                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
-                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
-            else:
-                endpoint_url, model, headers = resolve_task_endpoint(
-                    sess.endpoint_url, sess.model, sess.headers, owner=user
-                )
-        else:
-            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
-    
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
-
-        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
-        filename = file.filename or "upload"
-        _, ext = os.path.splitext(filename.lower())
-
-        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
-        if ext not in allowed:
-            raise HTTPException(400, f"Unsupported file type: {ext}")
-
-        # Extract text based on file type
-        if ext == ".pdf":
-            from src.document_processor import _process_pdf
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
-                tmp.write(content)
-                tmp_path = tmp.name
-            try:
-                text = _process_pdf(tmp_path, owner=_owner(request))
-            finally:
-                os.unlink(tmp_path)
-        else:
-            try:
-                text = content.decode("utf-8")
-            except UnicodeDecodeError:
-                from charset_normalizer import detect
-                encoding = (detect(content) or {}).get("encoding") or "utf-8"
-                text = content.decode(encoding, errors="replace")
-
-        if not text.strip():
-            return {"suggestions": [], "message": "No readable content found"}
-
-        # Fast path: a .json upload that already looks like a memories export
-        # (list of {text, category, ...} dicts, or list of strings) round-trips
-        # directly without spending an LLM call to re-extract its own output.
-        # Without this, re-importing a memories.json from another account
-        # ran the file through the extractor, which often re-emitted the
-        # entries as a numbered list (and the numbering leaked into the
-        # `text` field).
-        if ext == ".json":
-            try:
-                parsed = json.loads(text)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list) and parsed:
-                direct = []
-                for item in parsed:
-                    if isinstance(item, dict) and item.get("text"):
-                        direct.append({
-                            "text": _strip_list_prefix(str(item["text"])),
-                            "category": item.get("category") or "fact",
-                        })
-                    elif isinstance(item, str) and item.strip():
-                        direct.append({
-                            "text": _strip_list_prefix(item.strip()),
-                            "category": "fact",
-                        })
-                if direct:
-                    return {"suggestions": direct, "filename": filename}
-
-        # Truncate very long documents
-        if len(text) > 15000:
-            text = text[:15000] + "\n[Truncated]"
-
-        # Send to LLM for memory extraction
-        import_prompt = (
-            "You are a memory extraction assistant. The user uploaded a document. "
-            "Analyze the text below and extract specific, useful facts — things like "
-            "names, preferences, jobs, locations, relationships, opinions, projects, "
-            "goals, contacts, or any other personal details worth remembering.\n\n"
-            "Rules:\n"
-            "- Each fact should be a short, self-contained statement\n"
-            "- Do NOT extract generic knowledge\n"
-            "- Focus on personal, memorable information\n"
-            "- If there are no useful facts, return an empty array\n\n"
-            "Return a JSON array of objects with 'text' and 'category' fields.\n"
-            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
-            "Return ONLY valid JSON, no markdown fences."
-        )
-
-        try:
-            raw = await llm_call_async(
-                endpoint_url,
-                model,
-                [
-                    {"role": "system", "content": import_prompt},
-                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
-                ],
-                temperature=0.2,
-                max_tokens=2000,
-                headers=headers,
-            )
-
-            # Parse JSON
-            raw = raw.strip()
-            if raw.startswith("```"):
-                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-
-            suggestions = json.loads(raw)
-            if isinstance(suggestions, list):
-                normalized = []
-                for s in suggestions:
-                    if not s:
-                        continue
-                    if isinstance(s, dict):
-                        s = dict(s)
-                        if s.get("text"):
-                            s["text"] = _strip_list_prefix(str(s["text"]))
-                        normalized.append(s)
-                    else:
-                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
-                suggestions = normalized
-            else:
-                suggestions = []
-
-            return {"suggestions": suggestions, "filename": filename}
-
-        except json.JSONDecodeError:
-            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
-            # numbering the model added so saved memories don't keep the prefix.
-            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
-            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
-        except Exception as e:
-            logger.error(f"Memory import extraction failed: {e}")
-            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
-
-    @router.post("/{memory_id}/pin")
-    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
-        """Pin or unpin a memory. Pinned memories are always included in context."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["pinned"] = pinned
-                memory_manager.save(all_mem)
-                return {"ok": True, "pinned": pinned}
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
-    @router.get("/{memory_id}")
-    def get_memory_item(request: Request, memory_id: str):
-        """Get a specific memory item by ID."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        for memory in memories:
-            if memory["id"] == memory_id:
-                return {"memory": memory}
-
-        raise HTTPException(404, "Memory not found")
-
-    @router.put("/{memory_id}")
-    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
-        """Update an existing memory item with new text and optional category."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["text"] = text.strip()
-                if category:
-                    all_mem[i]["category"] = category
-                all_mem[i]["timestamp"] = int(time.time())
-
-                memory_manager.save(all_mem)
-                # Sync vector index (remove old, add updated)
-                if memory_vector and memory_vector.healthy:
-                    memory_vector.remove(memory_id)
-                    memory_vector.add(memory_id, text.strip())
-                return {"ok": True, "message": "Memory updated successfully"}
-
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    @router.delete("/{memory_id}")
-    def delete_memory(request: Request, memory_id: str):
-        """Delete a memory item by its ID."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-
-        # Find and verify ownership before deleting
-        target = next((m for m in all_mem if m["id"] == memory_id), None)
-        if not target:
-            raise HTTPException(404, f"Memory item {memory_id} not found")
-        _verify_memory_owner(target, user)
-
-        all_mem = [m for m in all_mem if m["id"] != memory_id]
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.remove(memory_id)
-        return {"ok": True, "message": "Memory deleted successfully"}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/model_routes.py

@@ -17,7 +17,9 @@ from fastapi import APIRouter, HTTPException, Form, Query, Body, Request, Respon
 from pydantic import BaseModel
 from fastapi.responses import StreamingResponse
 from core.database import SessionLocal, ModelEndpoint, Session as DbSession
+from core.log_safety import redact_url as _redact_url_for_log
 from core.middleware import require_admin
+from src.constants import COOKBOOK_STATE_FILE
 from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
 from src.tls_overrides import llm_verify
 from src.settings import load_settings as _load_settings, save_settings as _save_settings
@@ -111,6 +113,67 @@ def _clear_endpoint_settings_for_endpoint(settings: dict, ep_id: str, *, include
     return cleared
 
 
+_COOKBOOK_ACTIVE_SERVE_STATUSES = {
+    "starting", "loading", "ready", "running", "restarting",
+}
+
+
+def _active_cookbook_endpoint_ids() -> set[str]:
+    """Endpoint IDs owned by active Cookbook serve tasks.
+
+    Cookbook auto-registers endpoints with ids like ``local-*``. Those rows are
+    managed lifecycle state, not durable user configuration. If a tmux stream is
+    stopped or an old task lingers, the row must stop participating in model
+    selection and defaults.
+    """
+    try:
+        if not os.path.exists(COOKBOOK_STATE_FILE):
+            return set()
+        with open(COOKBOOK_STATE_FILE, "r", encoding="utf-8") as fh:
+            raw = fh.read()
+        state = json.loads(raw)
+    except Exception:
+        return set()
+    out: set[str] = set()
+    for task in state.get("tasks") or []:
+        if not isinstance(task, dict) or task.get("type") != "serve":
+            continue
+        if str(task.get("status") or "").lower() not in _COOKBOOK_ACTIVE_SERVE_STATUSES:
+            continue
+        ep_id = task.get("_endpointId") or task.get("endpointId") or task.get("endpoint_id")
+        if ep_id:
+            out.add(str(ep_id))
+    return out
+
+
+def _disable_stale_cookbook_local_endpoints(db) -> int:
+    """Disable enabled cookbook endpoints whose serve task is no longer active."""
+    active_ids = _active_cookbook_endpoint_ids()
+    if not active_ids:
+        return 0
+    stale = (
+        db.query(ModelEndpoint)
+        .filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+        .filter(ModelEndpoint.id.like("local-%"))
+        .filter(~ModelEndpoint.id.in_(active_ids))
+        .all()
+    )
+    if not stale:
+        return 0
+    settings = _load_settings()
+    touched_settings = False
+    for ep in stale:
+        ep.is_enabled = False
+        ep.model_refresh_mode = "disabled"
+        if _clear_endpoint_settings_for_endpoint(settings, ep.id):
+            touched_settings = True
+        logger.info("Disabled stale Cookbook endpoint %s (%s @ %s)", ep.id, ep.name, ep.base_url)
+    if touched_settings:
+        _save_settings(settings)
+    db.commit()
+    return len(stale)
+
+
 def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     """Remove endpoint references from scoped or legacy-flat user preferences."""
     if not isinstance(all_prefs, dict):
@@ -124,7 +187,24 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     return cleared_users
 
 
-def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool:
+def _endpoint_visible_model_ids(ep: Any) -> List[str]:
+    """Known visible model ids for an endpoint, including pinned/manual ids."""
+    if ep is None:
+        return []
+    return _visible_models(
+        getattr(ep, "cached_models", None),
+        getattr(ep, "hidden_models", None),
+        getattr(ep, "pinned_models", None),
+    )
+
+
+def _default_endpoint_needs_assignment(
+    current_default_id: str,
+    enabled_endpoint_ids,
+    *,
+    current_default_endpoint: Any = None,
+    current_default_model: str = "",
+) -> bool:
     """Whether the global default chat endpoint should be (re)assigned.
 
     True when nothing is configured yet, or the configured default no longer
@@ -136,7 +216,14 @@ def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint
     """
     if not current_default_id:
         return True
-    return current_default_id not in enabled_endpoint_ids
+    if current_default_id not in enabled_endpoint_ids:
+        return True
+    if current_default_endpoint is None:
+        return False
+    if not (current_default_model or "").strip():
+        return True
+    visible = _endpoint_visible_model_ids(current_default_endpoint)
+    return bool(visible and current_default_model not in visible)
 
 
 # Loopback hosts a user might type for a local model server (LM Studio,
@@ -406,8 +493,11 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float:
     except Exception:
         val = 0
     if val > 0:
-        return float(max(1, min(30, val)))
-    return 2.5 if category == "local" else 2.0
+        return float(max(1, min(60, val)))
+    # llama.cpp and other local OpenAI-compatible servers can block briefly
+    # while warming/loading. A 2s local timeout makes working endpoints flicker
+    # offline before /v1/models is ready.
+    return 10.0 if category == "local" else 2.0
 
 
 def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float:
@@ -474,7 +564,7 @@ def _explicit_model_list_timeout(base_url: str, endpoint_kind: str = "auto", req
     category = _classify_endpoint(base_url, kind)
     if kind in ("api", "proxy") or category == "api":
         return 30.0
-    return 3.0 if _is_ollama_base(base_url) else 2.0
+    return 15.0 if category == "local" else (3.0 if _is_ollama_base(base_url) else 2.0)
 
 
 def _cached_model_ids(ep: Any) -> List[str]:
@@ -519,6 +609,10 @@ _NON_CHAT_EXACT_PREFIXES = (
 
 def _is_chat_model(model_id: str) -> bool:
     """Return True if the model ID looks like a chat/completions-capable model."""
+    if not isinstance(model_id, str):
+        # Non-compliant upstreams can return non-string IDs (e.g. int/None);
+        # treat them as chat-capable rather than crashing on .lower().
+        return True
     mid = model_id.lower()
     for prefix in _NON_CHAT_PREFIXES:
         if mid.startswith(prefix):
@@ -711,6 +805,51 @@ def _effective_endpoint_kind(ep: Any, base_url: str) -> str:
     return "auto"
 
 
+def _is_loading_model_response(resp: Any) -> bool:
+    if getattr(resp, "status_code", None) != 503:
+        return False
+    try:
+        body = resp.text or ""
+    except Exception:
+        body = ""
+    return "loading model" in body.lower()
+
+
+
+def _openai_model_ids(data: Any) -> List[str]:
+    """Extract OpenAI-style model IDs.
+
+    Accepts both standard ``{"data": [{"id": ...}]}`` responses and bare
+    ``[{"id": ...}]`` lists returned by some OpenAI-compatible providers.
+    Tolerates non-dict/non-list bodies and non-string IDs, returning only
+    non-empty string IDs.
+    """
+    if isinstance(data, list):
+        items = data
+    elif isinstance(data, dict):
+        items = data.get("data")
+    else:
+        items = None
+    return [m["id"] for m in (items or [])
+            if isinstance(m, dict) and isinstance(m.get("id"), str) and m["id"]]
+
+
+def _ollama_model_names(data: Any) -> List[str]:
+    """Extract native-Ollama model names (``{"models": [{"name"|"model": ...}]}``).
+
+    Same tolerance as :func:`_openai_model_ids`: a non-dict body or non-string
+    value is skipped rather than crashing, preserving name-then-model precedence.
+    """
+    items = data.get("models") if isinstance(data, dict) else None
+    out: List[str] = []
+    for m in (items or []):
+        if not isinstance(m, dict):
+            continue
+        v = m.get("name") or m.get("model")
+        if isinstance(v, str) and v:
+            out.append(v)
+    return out
+
 
 def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]:
     """Probe a base URL's /models endpoint and return list of model IDs.
@@ -734,7 +873,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
             r = httpx.get(url, headers=headers, timeout=timeout, verify=llm_verify())
             r.raise_for_status()
             data = r.json()
-            models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
+            models = _openai_model_ids(data)
             if models:
                 return models
         except httpx.HTTPStatusError as e:
@@ -756,10 +895,10 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
         r.raise_for_status()
         data = r.json()
         # OpenAI format: {"data": [{"id": "model-name"}]}
-        models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
+        models = _openai_model_ids(data)
         # Ollama format: {"models": [{"name": "model-name"}]}
         if not models:
-            models = [m.get("name") or m.get("model") for m in (data.get("models") or []) if m.get("name") or m.get("model")]
+            models = _ollama_model_names(data)
         if models:
             # Z.AI coding plan omits some working models from /models;
             # append curated-only entries for that endpoint only.
@@ -775,16 +914,19 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
                         models.append(_e)
             return [m for m in models if _is_chat_model(m)]
     except httpx.HTTPStatusError as e:
+        if e.response is not None and _is_loading_model_response(e.response):
+            logger.info("Endpoint still loading model at %s", _redact_url_for_log(url))
+            return []
         if api_key:
             status = e.response.status_code if e.response is not None else "unknown"
-            logger.warning(f"Failed to probe {url} with API key: HTTP {status}")
+            logger.warning("Failed to probe %s with API key: HTTP %s", _redact_url_for_log(url), status)
             return []
-        logger.warning(f"Failed to probe {url}: {e}")
+        logger.warning("Failed to probe %s: %s", _redact_url_for_log(url), e)
     except Exception as e:
         if api_key:
-            logger.warning(f"Failed to probe {url} with API key: {e}")
+            logger.warning("Failed to probe %s with API key: %s", _redact_url_for_log(url), e)
             return []
-        logger.warning(f"Failed to probe {url}: {e}")
+        logger.warning("Failed to probe %s: %s", _redact_url_for_log(url), e)
 
     # Older Ollama builds and some proxies expose native /api/tags even when
     # the OpenAI-compatible /v1/models path is unavailable.
@@ -795,7 +937,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
             r = httpx.get(root + "/api/tags", timeout=timeout, verify=llm_verify())
             r.raise_for_status()
             data = r.json()
-            models = [m.get("name") or m.get("model") for m in (data.get("models") or []) if m.get("name") or m.get("model")]
+            models = _ollama_model_names(data)
             if models:
                 return [m for m in models if _is_chat_model(m)]
     except Exception as e:
@@ -824,6 +966,15 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
         or "ollama" in (parsed_base.hostname or "").lower()
     )
 
+    def _is_loading_model_response(r) -> bool:
+        if getattr(r, "status_code", None) != 503:
+            return False
+        try:
+            body = r.text or ""
+        except Exception:
+            body = ""
+        return "loading model" in body.lower()
+
     def _result_from_response(r) -> Dict[str, Any]:
         if 300 <= r.status_code < 400:
             loc = r.headers.get("location", "")
@@ -840,6 +991,13 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
                 "status_code": r.status_code,
                 "error": None,
             }
+        if _is_loading_model_response(r):
+            return {
+                "reachable": True,
+                "loading": True,
+                "status_code": r.status_code,
+                "error": "Loading model",
+            }
         return {"reachable": False, "status_code": r.status_code, "error": f"HTTP {r.status_code}"}
 
     last_error: Optional[str] = None
@@ -872,7 +1030,7 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
         if 400 <= sc < 500 and sc not in (401, 403):
             models_url = _safe_build_models_url(base)
             try:
-                r2 = httpx.get(models_url, headers=headers, timeout=timeout, verify=llm_verify())
+                r2 = httpx.get(models_url, headers=headers,timeout=timeout, verify=llm_verify())
                 result2 = _result_from_response(r2)
                 if result2["reachable"]:
                     return result2
@@ -1056,9 +1214,11 @@ def setup_model_routes(model_discovery):
         except Exception:
             return 0.0
 
-    def _failure_delay(fails: int) -> float:
+    def _failure_delay(fails: int, *, empty_local: bool = False) -> float:
         if fails <= 0:
             return 0.0
+        if empty_local:
+            return min(5.0 * (2 ** max(0, fails - 1)), 30.0)
         return min(_REFRESH_FAILURE_BASE * (2 ** max(0, fails - 1)), _REFRESH_FAILURE_MAX)
 
     def _should_refresh_endpoint(ep: Any, now: float, force: bool = False) -> tuple[bool, Dict[str, Any]]:
@@ -1089,7 +1249,12 @@ def setup_model_routes(model_discovery):
         fails = int(state.get("fail_count") or 0)
         if fails and not force:
             last_failure = float(state.get("last_failure") or 0.0)
-            if now - last_failure < _failure_delay(fails):
+            empty_local = (
+                not cached
+                and category == "local"
+                and str(getattr(ep, "id", "") or "").startswith("local-")
+            )
+            if now - last_failure < _failure_delay(fails, empty_local=empty_local):
                 return False, info
         if cached and not force:
             interval = _endpoint_refresh_interval(ep, category)
@@ -1115,6 +1280,8 @@ def setup_model_routes(model_discovery):
                 db = SessionLocal()
                 changed = False
                 try:
+                    if _disable_stale_cookbook_local_endpoints(db):
+                        changed = True
                     endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
                     now = _time.time()
                     groups: Dict[str, Dict[str, Any]] = {}
@@ -1188,6 +1355,8 @@ def setup_model_routes(model_discovery):
 
         db = SessionLocal()
         try:
+            if _disable_stale_cookbook_local_endpoints(db):
+                _invalidate_models_cache()
             q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
             if owner and not is_admin:
                 # Regular users see: their own endpoints + null-owner
@@ -1257,7 +1426,7 @@ def setup_model_routes(model_discovery):
         return {"hosts": [], "items": items}
 
     @router.get("/models")
-    def api_models(request: Request, refresh: bool = False):
+    def api_models(request: Request, refresh: bool = False, background: bool = True):
         """Get available models — per-user (caller sees only their endpoints +
         legacy/shared null-owner rows). Cached per-user for 30s."""
         # Require auth; "" is the unconfigured single-user mode, treated as
@@ -1299,8 +1468,11 @@ def setup_model_routes(model_discovery):
             return cache_entry["data"]
         result = _fetch_models(owner=owner, is_admin=_is_admin)
         _models_cache[_cache_key] = {"data": result, "time": now}
-        # Kick off background refresh to update caches from live endpoints
-        _refresh_caches_bg(force=refresh)
+        # Kick off background refresh to update caches from live endpoints.
+        # Page boot can opt out with background=false so opening Odysseus does
+        # not start endpoint probes against slow/offline model servers.
+        if background or refresh:
+            _refresh_caches_bg(force=refresh)
         return result
 
     # Brief cache for local-probe results so picker-open doesn't hammer
@@ -1309,6 +1481,7 @@ def setup_model_routes(model_discovery):
     # within ~8s of the user noticing.
     _LOCAL_PROBE_TTL = 8.0
     _local_probe_cache: Dict[str, Any] = {"data": None, "time": 0.0}
+    _local_probe_inflight: Dict[str, Any] = {"task": None}
 
     @router.get("/model-endpoints/probe-local")
     async def probe_local_endpoints(request: Request):
@@ -1323,58 +1496,72 @@ def setup_model_routes(model_discovery):
                 (now - _local_probe_cache["time"]) < _LOCAL_PROBE_TTL):
             return _local_probe_cache["data"]
 
-        db = SessionLocal()
-        try:
-            endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
-            local_eps = []
-            for ep in endpoints:
-                base = _normalize_base(ep.base_url)
-                kind = _effective_endpoint_kind(ep, base)
-                if _classify_endpoint(base, kind) == "local":
-                    local_eps.append((ep.id, base, ep.api_key))
-        finally:
-            db.close()
-
-        grouped: Dict[str, Dict[str, Any]] = {}
-        for ep_id, base, api_key in local_eps:
-            key = _refresh_key(base, api_key)
-            grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
-
-        async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
-            t0 = _time.time()
-            try:
-                import asyncio as _asyncio
-                # Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
-                # local vLLM endpoints on Tailscale links where the model
-                # server is still loading (Qwen3.5-122B takes 2–3 min to
-                # warm); /v1/models can take 500–2500 ms on a busy box,
-                # which pushed _ping_endpoint's full path-discovery sweep
-                # past the cap and marked the row offline despite the
-                # user actively chatting with it.
-                ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
-                lat = round((_time.time() - t0) * 1000)
-                return {
-                    "alive": bool(ping.get("reachable")),
-                    "latency_ms": lat,
-                    "status_code": ping.get("status_code"),
-                    "error": ping.get("error"),
-                }
-            except Exception as e:
-                return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
-
         import asyncio as _asyncio
-        results_list = await _asyncio.gather(
-            *[_probe_one(data) for data in grouped.values()],
-            return_exceptions=False,
-        )
-        results: Dict[str, Any] = {}
-        for data, r in zip(grouped.values(), results_list):
-            for eid in data["endpoint_ids"]:
-                results[eid] = r
+        task = _local_probe_inflight.get("task")
+        if task is not None and not task.done():
+            return await task
 
-        _local_probe_cache["data"] = results
-        _local_probe_cache["time"] = now
-        return results
+        async def _compute_local_probe() -> Dict[str, Any]:
+            db = SessionLocal()
+            try:
+                if _disable_stale_cookbook_local_endpoints(db):
+                    _invalidate_models_cache()
+                endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
+                local_eps = []
+                for ep in endpoints:
+                    base = _normalize_base(ep.base_url)
+                    kind = _effective_endpoint_kind(ep, base)
+                    if _classify_endpoint(base, kind) == "local":
+                        local_eps.append((ep.id, base, ep.api_key))
+            finally:
+                db.close()
+
+            grouped: Dict[str, Dict[str, Any]] = {}
+            for ep_id, base, api_key in local_eps:
+                key = _refresh_key(base, api_key)
+                grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
+
+            async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
+                t0 = _time.time()
+                try:
+                    # Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
+                    # local vLLM endpoints on Tailscale links where the model
+                    # server is still loading (Qwen3.5-122B takes 2–3 min to
+                    # warm); /v1/models can take 500–2500 ms on a busy box,
+                    # which pushed _ping_endpoint's full path-discovery sweep
+                    # past the cap and marked the row offline despite the
+                    # user actively chatting with it.
+                    ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
+                    lat = round((_time.time() - t0) * 1000)
+                    return {
+                        "alive": bool(ping.get("reachable")),
+                        "latency_ms": lat,
+                        "status_code": ping.get("status_code"),
+                        "error": ping.get("error"),
+                    }
+                except Exception as e:
+                    return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
+
+            results_list = await _asyncio.gather(
+                *[_probe_one(data) for data in grouped.values()],
+                return_exceptions=False,
+            )
+            results: Dict[str, Any] = {}
+            for data, r in zip(grouped.values(), results_list):
+                for eid in data["endpoint_ids"]:
+                    results[eid] = r
+
+            _local_probe_cache["data"] = results
+            _local_probe_cache["time"] = _time.time()
+            return results
+
+        task = _asyncio.create_task(_compute_local_probe())
+        _local_probe_inflight["task"] = task
+        try:
+            return await task
+        finally:
+            if _local_probe_inflight.get("task") is task:
+                _local_probe_inflight["task"] = None
 
     @router.get("/ping")
     def ping_endpoints(request: Request):
@@ -1404,7 +1591,7 @@ def setup_model_routes(model_discovery):
                 t0 = _time.time()
                 ping = _ping_endpoint(base, ep.api_key, timeout=1.5)
                 entry["latency_ms"] = round((_time.time() - t0) * 1000)
-                entry["status"] = "online" if ping.get("reachable") or cached_count else "offline"
+                entry["status"] = "loading" if ping.get("loading") else ("online" if ping.get("reachable") or cached_count else "offline")
                 entry["error"] = ping.get("error")
                 entry["model_count"] = cached_count or (len(ANTHROPIC_MODELS) if provider == "anthropic" else 0)
             except Exception as e:
@@ -1557,6 +1744,8 @@ def setup_model_routes(model_discovery):
         require_admin(request)
         db = SessionLocal()
         try:
+            if _disable_stale_cookbook_local_endpoints(db):
+                _invalidate_models_cache()
             rows = db.query(ModelEndpoint).order_by(ModelEndpoint.created_at).all()
             results = []
             for r in rows:
@@ -1564,39 +1753,11 @@ def setup_model_routes(model_discovery):
                 hidden = _hidden_model_ids(r)
                 pinned = _normalize_model_ids(getattr(r, "pinned_models", None))
                 visible = _visible_models(all_models, r.hidden_models, pinned)
-                # Endpoint counts as reachable if it has any model — including
-                # admin-pinned IDs that a probe would never surface.
-                status = "online" if (all_models or pinned) else "offline"
+                # Keep the list route cache-only. It feeds Settings →
+                # Added Models and must render immediately; explicit
+                # Refresh/Probe endpoints do the network work.
+                status = "online" if (all_models or pinned) else ("empty" if r.is_enabled else "offline")
                 ping = None
-                # When cached_models is empty, do a quick reachability probe.
-                # Bumped 1.0s → 3.5s because the user reported endpoints they
-                # were ACTIVELY chatting with showed "offline" — the previous
-                # 1s timeout was clipping live cloud endpoints (DeepSeek can
-                # take 1.5–2.5s on /v1/models when their region is under load,
-                # vLLM on a remote GPU box behind SSH can also push past 1s).
-                # 3.5s still keeps the picker render snappy in the common
-                # "everything's already cached" path because this branch only
-                # runs for endpoints with an empty cached_models.
-                if not all_models and not pinned and r.is_enabled:
-                    ping = _ping_endpoint(r.base_url, r.api_key, timeout=3.5)
-                    if ping.get("reachable"):
-                        status = "empty"
-                        # Best-effort: if the probe came back reachable, try
-                        # to populate cached_models in the background so the
-                        # NEXT picker load shows "online" instead of "empty".
-                        # Failure here is silent — we already returned the
-                        # "empty" status, and the existing background refresh
-                        # path will eventually fill it in too.
-                        try:
-                            probed = _probe_endpoint(r.base_url, r.api_key, timeout=5)
-                            if probed:
-                                r.cached_models = json.dumps(probed)
-                                db.commit()
-                                all_models = probed
-                                visible = _visible_models(all_models, r.hidden_models, pinned)
-                                status = "online"
-                        except Exception as _refill_err:
-                            logger.debug(f"opportunistic cached_models refill failed for {r.id}: {_refill_err!r}")
                 base = _normalize_base(r.base_url)
                 kind = _effective_endpoint_kind(r, base)
                 results.append({
@@ -1766,7 +1927,7 @@ def setup_model_routes(model_discovery):
         model_ids = _probe_endpoint(base_url, api_key.strip() or None, timeout=explicit_timeout) if should_probe else []
         ping = {"reachable": False, "error": None}
         if (should_probe or requested_kind in ("api", "proxy")) and not model_ids:
-            ping = _ping_endpoint(base_url, api_key.strip() or None, timeout=min(explicit_timeout, 2.0))
+            ping = _ping_endpoint(base_url, api_key.strip() or None, timeout=min(explicit_timeout, 10.0))
         if require_model_list and not model_ids:
             raise HTTPException(400, _model_endpoint_error_message(base_url, ping))
 
@@ -1813,7 +1974,18 @@ def setup_model_routes(model_discovery):
                     ModelEndpoint.is_enabled == True  # noqa: E712
                 ).all()
             }
-            if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids):
+            current_default_id = settings.get("default_endpoint_id") or ""
+            current_default_ep = None
+            if current_default_id:
+                current_default_ep = db.query(ModelEndpoint).filter(
+                    ModelEndpoint.id == current_default_id
+                ).first()
+            if _default_endpoint_needs_assignment(
+                current_default_id,
+                enabled_ids,
+                current_default_endpoint=current_default_ep,
+                current_default_model=settings.get("default_model") or "",
+            ):
                 from src.endpoint_resolver import _first_chat_model
                 settings["default_endpoint_id"] = ep.id
                 settings["default_model"] = _first_chat_model(model_ids) or ""
@@ -1833,7 +2005,7 @@ def setup_model_routes(model_discovery):
             "models": _merge_model_ids(model_ids, _pinned),
             "pinned_models": _pinned,
             "online": bool(model_ids) or bool(_pinned) or bool(ping.get("reachable")),
-            "status": "online" if (model_ids or _pinned) else ("empty" if ping.get("reachable") else "offline"),
+            "status": "online" if (model_ids or _pinned) else ("loading" if ping.get("loading") else ("empty" if ping.get("reachable") else "offline")),
             "ping_error": ping.get("error") if ping else None,
             "endpoint_kind": requested_kind,
             "category": _classify_endpoint(base_url, requested_kind),
@@ -1858,11 +2030,11 @@ def setup_model_routes(model_discovery):
         configured_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60)
         probe_timeout = _explicit_model_list_timeout(base_url, requested_kind, configured_timeout)
         models = _probe_endpoint(base_url, api_key.strip() or None, timeout=probe_timeout)
-        ping = {"reachable": True, "error": None} if models else _ping_endpoint(base_url, api_key.strip() or None, timeout=min(probe_timeout, 2.0))
+        ping = {"reachable": True, "error": None} if models else _ping_endpoint(base_url, api_key.strip() or None, timeout=min(probe_timeout, 10.0))
         return {
             "base_url": base_url,
             "online": bool(models) or bool(ping.get("reachable")),
-            "status": "online" if models else ("empty" if ping.get("reachable") else "offline"),
+            "status": "online" if models else ("loading" if ping.get("loading") else ("empty" if ping.get("reachable") else "offline")),
             "ping_error": ping.get("error") if ping else None,
             "models": models,
             "count": len(models),
@@ -2040,6 +2212,16 @@ def setup_model_routes(model_discovery):
             ep_id = (_user_prefs.get("default_endpoint_id") or "").strip()
             model = (_user_prefs.get("default_model") or "").strip()
             _fallbacks = _user_prefs.get("default_model_fallbacks") or []
+            # If user has no personal default, fall back to global default
+            # But only based on the "share_defaults_with_users" flag
+            # (only if share_defaults_with_users is enabled)
+            if settings.get("share_defaults_with_users", False):
+                if not ep_id:
+                    ep_id = settings.get("default_endpoint_id", "")
+                if not model:
+                    model = settings.get("default_model", "")
+                if not _fallbacks:
+                    _fallbacks = settings.get("default_model_fallbacks") or []
         else:
             ep_id = settings.get("default_endpoint_id", "")
             model = settings.get("default_model", "")

routes/note_routes.py

@@ -335,10 +335,11 @@ async def dispatch_reminder(
             # Loud diagnostic so we can see WHY a reminder didn't send (the
             # previous "silently no-op when cfg has no smtp_host" was invisible).
             logger.info(
-                f"dispatch_reminder[email] note_id={note_id} owner={owner!r} "
-                f"smtp_host={cfg.get('smtp_host')!r} smtp_user={cfg.get('smtp_user')!r} "
-                f"from={from_addr!r} recipient={recipient!r} "
-                f"account_name={cfg.get('account_name')!r}"
+                "dispatch_reminder[email] note_id=%s owner=%r "
+                "has_smtp_host=%s has_smtp_user=%s has_from=%s has_recipient=%s",
+                note_id, owner,
+                bool(cfg.get("smtp_host")), bool(cfg.get("smtp_user")),
+                bool(from_addr), bool(recipient),
             )
             missing = []
             if not cfg.get("smtp_host"):

routes/preset_routes.py

@@ -1,5 +1,6 @@
 """Preset routes — /api/presets GET, /api/presets/custom POST, user templates CRUD."""
 
+import asyncio
 import logging
 import uuid
 from typing import Dict, Any, List
@@ -102,7 +103,7 @@ def setup_preset_routes(preset_manager) -> APIRouter:
         try:
             model_spec = data.get("model") or ""
             user = effective_user(request)
-            url, model, headers = _resolve_model(model_spec, owner=user)
+            url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=user)
             result = await llm_call_async(url, model, messages, temperature=0.8, max_tokens=500, headers=headers)
             return {"success": True, "prompt": result.strip()}
         except Exception as e:

routes/research/__init__.py

@@ -0,0 +1,5 @@
+"""Research route domain package (slice 2b, #4082/#4071).
+
+Contains research_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/research_routes.py re-exports from here.
+"""

routes/research/research_routes.py

@@ -0,0 +1,749 @@
+"""Research background task routes — /api/research/*."""
+
+import asyncio
+import json
+import logging
+import re
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import HTMLResponse, StreamingResponse
+from pydantic import BaseModel, Field
+from core.middleware import INTERNAL_TOOL_USER
+from src.endpoint_resolver import resolve_endpoint
+from src.auth_helpers import _auth_disabled, get_current_user
+from core.auth import RESERVED_USERNAMES
+from src.constants import DEEP_RESEARCH_DIR
+
+_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
+
+
+def _confine_research_path(session_id: str) -> Path:
+    """Return the resolved Path for session_id's JSON inside DEEP_RESEARCH_DIR.
+
+    Validates the session ID format and asserts containment after symlink
+    expansion. Raises HTTPException(400) on format failures, traversal
+    attempts, absolute-path injection, and symlink escape so every caller
+    gets a safe, confined path with no extra validation needed.
+    """
+    if not _SESSION_ID_RE.fullmatch(session_id):
+        raise HTTPException(400, "Invalid session ID")
+    root = Path(DEEP_RESEARCH_DIR).resolve()
+    candidate = (root / f"{session_id}.json").resolve()
+    try:
+        candidate.relative_to(root)
+    except ValueError:
+        raise HTTPException(400, "Invalid session ID")
+    return candidate
+
+
+logger = logging.getLogger(__name__)
+
+# Model-name substrings that are NOT chat/generation models — research must
+# never pick these as its model. An OpenAI-style endpoint often lists
+# `text-embedding-ada-002` etc. first in its model list, which is why research
+# was failing with "Cannot reach model 'text-embedding-ada-002'".
+_NON_CHAT_MODEL = (
+    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
+    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
+)
+
+_RESEARCH_IMAGE_BLOCKLIST = {
+    "cdn.shopify.com/s/files/1/0179/4388/7926/files/icon.png",
+}
+
+
+def _is_research_icon_or_logo_url(url: str) -> bool:
+    path = url.lower().split("?")[0]
+    return any(token in path for token in (
+        "/logo", "logo_", "-logo", "favicon", "apple-touch-icon",
+        "sprite", "icon-", "_icon", "/icons/", "badge",
+    ))
+
+
+def _research_thumbnail(data: dict) -> str:
+    """Pick the same first visible image the visual report uses as hero."""
+    hidden = set(data.get("hidden_images") or [])
+    seen = set()
+
+    def usable(image: str) -> bool:
+        image = str(image or "").strip()
+        if not image or image in seen or image in hidden:
+            return False
+        if not image.startswith("https://"):
+            return False
+        if image.endswith((".svg", ".ico", ".gif")):
+            return False
+        if any(blocked in image for blocked in _RESEARCH_IMAGE_BLOCKLIST):
+            return False
+        if _is_research_icon_or_logo_url(image):
+            return False
+        return True
+
+    for source in data.get("sources") or []:
+        if not isinstance(source, dict):
+            continue
+        image = str(source.get("image") or source.get("og_image") or "").strip()
+        if usable(image):
+            seen.add(image)
+            return image
+    for finding in data.get("raw_findings") or data.get("findings") or []:
+        if not isinstance(finding, dict):
+            continue
+        image = str(finding.get("image") or finding.get("og_image") or "").strip()
+        if usable(image):
+            seen.add(image)
+            return image
+    return ""
+
+
+def _first_chat_model(models) -> str:
+    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
+    for m in (models or []):
+        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
+            return m
+    return (models[0] if models else "")
+
+
+def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
+    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
+    owner = owner or getattr(sess, "owner", None) or None
+    url, model, headers = resolve_endpoint(
+        "research",
+        fallback_url=sess.endpoint_url,
+        fallback_model=sess.model,
+        fallback_headers=sess.headers,
+        owner=owner,
+    )
+    return url, model, headers
+
+
+def _owned_enabled_endpoint(db, owner, endpoint_id=None):
+    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
+    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
+    None if nothing visible matches.
+
+    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
+    owner = private, "the model picker only shows the endpoint to that user") and
+    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
+    api_key + base_url into research_handler.start_research(llm_endpoint=,
+    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
+    via the bare first-enabled fallback — would let a research-privileged user
+    spend ANOTHER user's API key/quota and reach whatever internal base_url they
+    configured. Mirrors webhook_routes._first_enabled_endpoint and
+    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
+    legacy mode).
+    """
+    from src.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if endpoint_id:
+        q = q.filter(ModelEndpoint.id == endpoint_id)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
+    """Resolve a ModelEndpoint row into (chat_url, model, headers).
+
+    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
+    panel-selected research endpoints. ChatGPT Subscription endpoints keep
+    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
+    """
+    from src.endpoint_resolver import (
+        build_chat_url,
+        build_headers,
+        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
+    )
+
+    try:
+        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
+    except Exception as e:
+        logger.warning("Could not resolve endpoint credentials for research: %s", e)
+        return None
+
+    ep_model = (model or "").strip()
+    if not ep_model:
+        try:
+            models = json.loads(ep.cached_models) if ep.cached_models else []
+            if models:
+                ep_model = _first_chat_model(models)
+        except Exception:
+            pass
+    if not ep_model:
+        return None
+    return build_chat_url(base), ep_model, build_headers(api_key, base)
+
+
+def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
+    router = APIRouter(tags=["research"])
+
+    def _require_user(request: Request) -> str:
+        """All research endpoints require an authenticated user. Research
+        data isn't owner-scoped in the on-disk JSON yet, so we at least
+        block anonymous access. Multi-tenant deploys should additionally
+        verify the session belongs to this user."""
+        user = get_current_user(request)
+        if not user:
+            if _auth_disabled():
+                return ""
+            raise HTTPException(401, "Not authenticated")
+        return user
+
+    def _validate_session_id(session_id: str) -> None:
+        if not _SESSION_ID_RE.fullmatch(session_id):
+            raise HTTPException(400, "Invalid session ID format")
+
+    def _owns_in_memory(session_id: str, user: str) -> bool:
+        """Ownership check for an in-flight (in-memory) research task.
+        Falls back to the on-disk JSON if the task has already finished."""
+        entry = research_handler._active_tasks.get(session_id)
+        if entry is not None:
+            return entry.get("owner", "") == user
+        # Task no longer in memory — check the persisted JSON.
+        try:
+            path = _confine_research_path(session_id)
+        except HTTPException:
+            return False
+        if not path.exists():
+            return False
+        try:
+            return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
+        except Exception:
+            return False
+
+    @router.get("/api/research/active")
+    async def research_active(request: Request):
+        """List all currently active (running) research tasks."""
+        user = _require_user(request)
+        active = []
+        for sid, entry in research_handler._active_tasks.items():
+            # SECURITY: only show this user's running tasks.
+            if entry.get("owner", "") != user:
+                continue
+            if entry.get("status") == "running":
+                active.append({
+                    "session_id": sid,
+                    "query": entry.get("query", ""),
+                    "status": "running",
+                    "progress": entry.get("progress", {}),
+                    "started_at": entry.get("started_at", 0),
+                })
+        return {"active": active}
+
+    @router.get("/api/research/status/{session_id}")
+    async def research_status(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        status = research_handler.get_status(session_id)
+        if status is None:
+            raise HTTPException(404, "No research found for this session")
+        return status
+
+    @router.post("/api/research/cancel/{session_id}")
+    async def research_cancel(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        cancelled = research_handler.cancel_research(session_id)
+        return {"cancelled": cancelled}
+
+    @router.post("/api/research/result/{session_id}")
+    async def research_result(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research result available")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        research_handler.clear_result(session_id)
+        return {"result": result, "sources": sources, "raw_findings": raw_findings}
+
+    def _assert_owns_research(session_id: str, user: str) -> None:
+        """404-not-403 ownership gate for a research session's on-disk JSON.
+        Use BEFORE returning any data or mutating the file."""
+        path = _confine_research_path(session_id)
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
+        except Exception:
+            raise HTTPException(404, "Research not found")
+        if owner != user:
+            raise HTTPException(404, "Research not found")
+
+    @router.get("/api/research/report/{session_id}")
+    async def research_report(session_id: str, request: Request):
+        """Serve the visual HTML report for a completed research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        logger.info(f"Visual report requested for session {session_id}")
+        try:
+            html_content = research_handler.get_report_html(session_id)
+        except Exception as e:
+            logger.error(f"Visual report generation error: {e}", exc_info=True)
+            raise HTTPException(500, f"Report generation failed: {e}")
+        if html_content is None:
+            logger.warning(f"No report data found for session {session_id}")
+            raise HTTPException(404, "No visual report available for this session")
+        return HTMLResponse(content=html_content)
+
+    class HideImageRequest(BaseModel):
+        url: str
+
+    @router.post("/api/research/{session_id}/hide-image")
+    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
+        """Mark an image URL as hidden for this research's visual report.
+        Persisted to the research JSON so subsequent /report renders skip it."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.hide_image(session_id, body.url)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.post("/api/research/{session_id}/unhide-images")
+    async def research_unhide_images(session_id: str, request: Request):
+        """Clear the hidden-images list for a research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.unhide_all_images(session_id)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.get("/api/research/library")
+    async def research_library(
+        request: Request,
+        search: Optional[str] = Query(None),
+        sort: str = Query("recent"),
+        limit: int = Query(50),
+        archived: bool = Query(False),
+    ):
+        user = _require_user(request)
+        """List all completed research for the Library panel."""
+        data_dir = Path(DEEP_RESEARCH_DIR)
+        items = []
+        for p in data_dir.glob("*.json"):
+            try:
+                d = json.loads(p.read_text(encoding="utf-8"))
+                # SECURITY: only show research belonging to this user. Legacy
+                # JSONs without an `owner` field are hidden — auth was the only
+                # gate before, so every user saw every other user's reports.
+                if d.get("owner") != user:
+                    continue
+                # Archived view shows ONLY archived reports; default hides them.
+                if bool(d.get("archived")) != archived:
+                    continue
+                query = d.get("query", "")
+                if search and search.lower() not in query.lower():
+                    continue
+                sources = d.get("sources", [])
+                items.append({
+                    "id": p.stem,
+                    "query": query,
+                    "category": d.get("category") or "",
+                    "source_count": len(sources),
+                    "status": d.get("status", "done"),
+                    "duration": d.get("stats", {}).get("Duration", ""),
+                    "rounds": d.get("stats", {}).get("Rounds", ""),
+                    "started_at": d.get("started_at", 0),
+                    "completed_at": d.get("completed_at", 0),
+                    "archived": bool(d.get("archived")),
+                    "thumbnail": _research_thumbnail(d),
+                })
+            except Exception:
+                continue
+
+        # Sort
+        if sort == "recent":
+            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
+        elif sort == "oldest":
+            items.sort(key=lambda x: x["completed_at"] or 0)
+        elif sort == "most-messages":
+            items.sort(key=lambda x: x["source_count"], reverse=True)
+        elif sort == "alpha":
+            items.sort(key=lambda x: x["query"].lower())
+
+        return {"research": items[:limit], "total": len(items)}
+
+    @router.get("/api/research/detail/{session_id}")
+    async def research_detail(session_id: str, request: Request):
+        """Return the full JSON for a single research result — sources,
+        summary, stats — used by the Library preview panel."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = _confine_research_path(session_id)
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as e:
+            raise HTTPException(500, f"Failed to read research: {e}")
+        # SECURITY: 404 (not 403) so we don't leak that the report exists.
+        if data.get("owner") != user:
+            raise HTTPException(404, "Research not found")
+        return data
+
+    @router.post("/api/research/{session_id}/archive")
+    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
+        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = _confine_research_path(session_id)
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("owner") != user:
+                raise HTTPException(404, "Research not found")
+            data["archived"] = bool(archived)
+            path.write_text(json.dumps(data), encoding="utf-8")
+        except HTTPException:
+            raise
+        except Exception as e:
+            raise HTTPException(500, f"Failed to update research: {e}")
+        return {"ok": True, "id": session_id, "archived": bool(archived)}
+
+    @router.delete("/api/research/{session_id}")
+    async def research_delete(session_id: str, request: Request):
+        """Delete a research result from disk."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        json_path = _confine_research_path(session_id)
+        deleted = False
+        if json_path.exists():
+            # SECURITY: verify ownership before letting the caller delete it.
+            try:
+                data = json.loads(json_path.read_text(encoding="utf-8"))
+                if data.get("owner") != user:
+                    raise HTTPException(404, "Research not found")
+            except HTTPException:
+                raise
+            except Exception:
+                raise HTTPException(404, "Research not found")
+            json_path.unlink()
+            deleted = True
+        return {"deleted": deleted}
+
+    # ------------------------------------------------------------------
+    # Panel endpoints — launch research without a chat session
+    # ------------------------------------------------------------------
+
+    class ResearchStartRequest(BaseModel):
+        query: str
+        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
+        max_rounds: int = Field(default=0, ge=0, le=20)
+        search_provider: Optional[str] = None
+        endpoint_id: Optional[str] = None
+        model: Optional[str] = None
+        max_time: int = Field(default=300, ge=60, le=1800)
+        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
+        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
+        category: Optional[str] = None
+
+    @router.post("/api/research/start")
+    async def research_start(body: ResearchStartRequest, request: Request):
+        """Launch a research job from the dedicated panel."""
+        from src.auth_helpers import require_privilege
+        user = require_privilege(request, "can_use_research")
+        if user == INTERNAL_TOOL_USER:
+            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
+            if tool_owner and tool_owner not in RESERVED_USERNAMES:
+                auth_mgr = getattr(request.app.state, "auth_manager", None)
+                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
+                    try:
+                        privs = auth_mgr.get_privileges(tool_owner) or {}
+                        if not privs.get("can_use_research", True):
+                            raise HTTPException(403, f"Your account is not allowed to can use research.")
+                    except HTTPException:
+                        raise
+                    except Exception:
+                        pass
+                user = tool_owner
+        session_id = f"rp-{uuid.uuid4().hex[:12]}"
+
+        if body.endpoint_id:
+            from src.database import SessionLocal
+            db = SessionLocal()
+            try:
+                # Owner-scoped: never resolve another user's private endpoint
+                # (and its decrypted api_key / internal base_url). A scoped miss
+                # reads as 404 so the endpoint's existence isn't revealed.
+                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
+                if not ep:
+                    raise HTTPException(404, "Endpoint not found or disabled")
+                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
+                if not resolved:
+                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
+                ep_url, ep_model, ep_headers = resolved
+            finally:
+                db.close()
+        else:
+            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
+            # When neither research nor utility is configured, use the user's
+            # configured DEFAULT model (default_endpoint_id/default_model) rather
+            # than arbitrarily grabbing the first enabled endpoint's first model
+            # (which surfaced gpt-3.5). "Default" should mean the default model.
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
+            if not ep_url:
+                from src.database import SessionLocal
+                db = SessionLocal()
+                try:
+                    # Owner-scoped first-enabled fallback: the caller's own rows
+                    # + legacy null-owner shared rows only — never borrow another
+                    # user's private endpoint/api_key. Same fix as the
+                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
+                    ep = _owned_enabled_endpoint(db, user)
+                    if ep:
+                        resolved = _resolve_endpoint_runtime(ep, owner=user)
+                        if resolved:
+                            ep_url, ep_model, ep_headers = resolved
+                finally:
+                    db.close()
+            if not ep_url:
+                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
+            if body.model:
+                ep_model = body.model
+
+        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
+        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
+        research_handler.start_research(
+            session_id=session_id,
+            query=body.query,
+            llm_endpoint=ep_url,
+            llm_model=ep_model,
+            max_time=body.max_time,
+            llm_headers=ep_headers,
+            max_rounds=effective_max_rounds,
+            search_provider=body.search_provider or None,
+            category=body.category or None,
+            extraction_timeout=body.extraction_timeout,
+            extraction_concurrency=body.extraction_concurrency,
+            owner=user,
+        )
+        return {"session_id": session_id, "status": "running", "query": body.query}
+
+    @router.get("/api/research/stream/{session_id}")
+    async def research_stream(session_id: str, request: Request):
+        """SSE stream of research progress events."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        async def _generate():
+            last_progress = None
+            while True:
+                status = research_handler.get_status(session_id)
+                if status is None:
+                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
+                    return
+                st = status.get("status", "")
+                progress = status.get("progress", {})
+                if progress != last_progress:
+                    last_progress = progress
+                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
+                if st != "running":
+                    final = {'status': st, 'final': True}
+                    task = research_handler._active_tasks.get(session_id, {})
+                    if st == "error" and task.get("result"):
+                        final['error'] = str(task["result"])[:500]
+                    yield f"data: {json.dumps(final)}\n\n"
+                    return
+                await asyncio.sleep(1.5)
+
+        return StreamingResponse(
+            _generate(),
+            media_type="text/event-stream",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
+    @router.post("/api/research/result-peek/{session_id}")
+    async def research_result_peek(session_id: str, request: Request):
+        """Get research result without clearing it (for panel use)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            p = _confine_research_path(session_id)
+            if p.exists():
+                d = json.loads(p.read_text(encoding="utf-8"))
+                return {
+                    "result": d.get("result", ""),
+                    "sources": d.get("sources", []),
+                    "raw_findings": d.get("raw_findings", []),
+                    "category": d.get("category") or "",
+                }
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
+
+    @router.post("/api/research/spinoff/{session_id}")
+    async def research_spinoff(session_id: str, request: Request):
+        """Create a new chat session pre-seeded with this research as context.
+
+        Reads the persisted research result + sources for `session_id`, creates
+        a fresh session (inheriting endpoint/model/headers from the source
+        session if available, otherwise from the resolved chat endpoint), and
+        injects a single system message containing the report and sources so
+        the user can ask follow-up questions in a clean conversation.
+        """
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        # SECURITY: gate on ownership before reading the persisted research —
+        # otherwise any authenticated user could spin off (and thereby read)
+        # another user's report by guessing its session ID. Mirrors every other
+        # endpoint in this file (see result_peek above).
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        if session_manager is None:
+            raise HTTPException(500, "session_manager not configured")
+
+        # Load research data — prefer in-memory result, fall back to disk
+        result = research_handler.get_result(session_id)
+        sources = research_handler.get_sources(session_id) or []
+        query = ""
+
+        path = _confine_research_path(session_id)
+        if path.exists():
+            try:
+                disk = json.loads(path.read_text(encoding="utf-8"))
+                if not result:
+                    result = disk.get("result")
+                if not sources:
+                    sources = disk.get("sources", []) or []
+                query = disk.get("query", "") or ""
+            except Exception as e:
+                logger.warning(f"Could not read research JSON for spinoff: {e}")
+
+        if not result:
+            raise HTTPException(404, "No research result available for this session")
+
+        # Inherit endpoint/model/headers from the source session when possible.
+        # For panel-launched research (rp-* IDs), there is no chat session, so
+        # fall back through the same chain as /api/research/start: research →
+        # utility → first enabled endpoint in the DB.
+        ep_url, ep_model, ep_headers = "", "", {}
+        try:
+            src_sess = session_manager.get_session(session_id)
+            ep_url = src_sess.endpoint_url or ""
+            ep_model = src_sess.model or ""
+            ep_headers = dict(src_sess.headers or {})
+        except KeyError:
+            pass
+
+        def _merge(r_url, r_model, r_headers):
+            nonlocal ep_url, ep_model, ep_headers
+            if not ep_url and r_url:
+                ep_url = r_url
+            if not ep_model and r_model:
+                ep_model = r_model
+            if not ep_headers and r_headers:
+                ep_headers = dict(r_headers)
+
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("chat", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("research", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("utility", owner=user))
+        if not ep_url or not ep_model:
+            # Last resort: this user's enabled endpoint, plus legacy shared rows.
+            from src.database import SessionLocal
+            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
+            db = SessionLocal()
+            try:
+                ep = _owned_enabled_endpoint(db, user)
+                if ep:
+                    base = normalize_base(ep.base_url)
+                    fallback_url = build_chat_url(base)
+                    fallback_headers = build_headers(ep.api_key, base)
+                    fallback_model = ""
+                    if ep.cached_models:
+                        try:
+                            models = json.loads(ep.cached_models)
+                            if models:
+                                fallback_model = _first_chat_model(models)
+                        except Exception:
+                            pass
+                    _merge(fallback_url, fallback_model, fallback_headers)
+            finally:
+                db.close()
+
+        if not ep_url or not ep_model:
+            raise HTTPException(400, "No endpoint configured — add one in Settings first")
+
+        # Create new session
+        new_sid = str(uuid.uuid4())
+
+        title_query = (query or "research").strip()
+        if len(title_query) > 60:
+            title_query = title_query[:57] + "…"
+        new_name = f"Follow-up: {title_query}"
+
+        new_sess = session_manager.create_session(
+            session_id=new_sid,
+            name=new_name,
+            endpoint_url=ep_url,
+            model=ep_model,
+            rag=False,
+            owner=user,
+        )
+        if ep_headers:
+            new_sess.headers = ep_headers
+            session_manager.save_sessions()
+        try:
+            from src.event_bus import fire_event
+            fire_event("session_created", user)
+        except Exception:
+            logger.debug("session_created event dispatch failed", exc_info=True)
+
+        # Build the priming system message — report only, no sources injected.
+        # The user can open the visual report for source details; keeping sources
+        # out of the chat context saves tokens and avoids the AI fabricating
+        # citations.
+        date_str = datetime.utcnow().strftime("%Y-%m-%d")
+        primer = (
+            f"[Research context — {date_str}]\n\n"
+            f"The user previously ran a deep research investigation. Use the "
+            f"report below as your primary knowledge base when answering "
+            f"follow-up questions. If the user asks something not covered, "
+            f"say so plainly rather than guessing.\n\n"
+            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
+            f"=== REPORT ===\n{result}"
+        )
+
+        from core.models import ChatMessage
+        new_sess.add_message(ChatMessage(
+            role="system",
+            content=primer,
+            metadata={"research_spinoff_from": session_id},
+        ))
+        session_manager.save_sessions()
+
+        return {
+            "session_id": new_sid,
+            "name": new_name,
+            "source_count": len(sources),
+        }
+
+    return router

routes/research_routes.py

@@ -1,678 +1,17 @@
-"""Research background task routes — /api/research/*."""
+"""Backward-compat shim — canonical location is routes/research/research_routes.py.
 
-import asyncio
-import json
-import logging
-import re
-import uuid
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.research_routes``, ``from routes.research_routes import X``,
+``importlib.import_module("routes.research_routes")``, and
+``monkeypatch.setattr("routes.research_routes.ATTR", ...)`` (string-targeted
+patch used by ``test_research_owner_scope_routes.py``) all operate on the
+*same* object the application actually uses. Keeps existing import paths
+working after slice 2b (#4082/#4071). Source-introspection tests read the
+canonical file by path.
+"""
 
-from fastapi import APIRouter, HTTPException, Query, Request
-from fastapi.responses import HTMLResponse, StreamingResponse
-from pydantic import BaseModel, Field
-from core.middleware import INTERNAL_TOOL_USER
-from src.endpoint_resolver import resolve_endpoint
-from src.auth_helpers import _auth_disabled, get_current_user
-from core.auth import RESERVED_USERNAMES
-from src.constants import DEEP_RESEARCH_DIR
+import sys as _sys
 
-_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
+from routes.research import research_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-# Model-name substrings that are NOT chat/generation models — research must
-# never pick these as its model. An OpenAI-style endpoint often lists
-# `text-embedding-ada-002` etc. first in its model list, which is why research
-# was failing with "Cannot reach model 'text-embedding-ada-002'".
-_NON_CHAT_MODEL = (
-    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
-    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
-)
-
-
-def _first_chat_model(models) -> str:
-    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
-    for m in (models or []):
-        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
-            return m
-    return (models[0] if models else "")
-
-
-def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
-    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
-    owner = owner or getattr(sess, "owner", None) or None
-    url, model, headers = resolve_endpoint(
-        "research",
-        fallback_url=sess.endpoint_url,
-        fallback_model=sess.model,
-        fallback_headers=sess.headers,
-        owner=owner,
-    )
-    return url, model, headers
-
-
-def _owned_enabled_endpoint(db, owner, endpoint_id=None):
-    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
-    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
-    None if nothing visible matches.
-
-    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
-    owner = private, "the model picker only shows the endpoint to that user") and
-    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
-    api_key + base_url into research_handler.start_research(llm_endpoint=,
-    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
-    via the bare first-enabled fallback — would let a research-privileged user
-    spend ANOTHER user's API key/quota and reach whatever internal base_url they
-    configured. Mirrors webhook_routes._first_enabled_endpoint and
-    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
-    legacy mode).
-    """
-    from src.database import ModelEndpoint
-    from src.auth_helpers import owner_filter
-    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
-    if endpoint_id:
-        q = q.filter(ModelEndpoint.id == endpoint_id)
-    return owner_filter(q, ModelEndpoint, owner).first()
-
-
-def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
-    """Resolve a ModelEndpoint row into (chat_url, model, headers).
-
-    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
-    panel-selected research endpoints. ChatGPT Subscription endpoints keep
-    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
-    """
-    from src.endpoint_resolver import (
-        build_chat_url,
-        build_headers,
-        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
-    )
-
-    try:
-        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
-    except Exception as e:
-        logger.warning("Could not resolve endpoint credentials for research: %s", e)
-        return None
-
-    ep_model = (model or "").strip()
-    if not ep_model:
-        try:
-            models = json.loads(ep.cached_models) if ep.cached_models else []
-            if models:
-                ep_model = _first_chat_model(models)
-        except Exception:
-            pass
-    if not ep_model:
-        return None
-    return build_chat_url(base), ep_model, build_headers(api_key, base)
-
-
-def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
-    router = APIRouter(tags=["research"])
-
-    def _require_user(request: Request) -> str:
-        """All research endpoints require an authenticated user. Research
-        data isn't owner-scoped in the on-disk JSON yet, so we at least
-        block anonymous access. Multi-tenant deploys should additionally
-        verify the session belongs to this user."""
-        user = get_current_user(request)
-        if not user:
-            if _auth_disabled():
-                return ""
-            raise HTTPException(401, "Not authenticated")
-        return user
-
-    def _validate_session_id(session_id: str) -> None:
-        if not _SESSION_ID_RE.fullmatch(session_id):
-            raise HTTPException(400, "Invalid session ID format")
-
-    def _owns_in_memory(session_id: str, user: str) -> bool:
-        """Ownership check for an in-flight (in-memory) research task.
-        Falls back to the on-disk JSON if the task has already finished."""
-        entry = research_handler._active_tasks.get(session_id)
-        if entry is not None:
-            return entry.get("owner", "") == user
-        # Task no longer in memory — check the persisted JSON.
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            return False
-        try:
-            return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
-        except Exception:
-            return False
-
-    @router.get("/api/research/active")
-    async def research_active(request: Request):
-        """List all currently active (running) research tasks."""
-        user = _require_user(request)
-        active = []
-        for sid, entry in research_handler._active_tasks.items():
-            # SECURITY: only show this user's running tasks.
-            if entry.get("owner", "") != user:
-                continue
-            if entry.get("status") == "running":
-                active.append({
-                    "session_id": sid,
-                    "query": entry.get("query", ""),
-                    "status": "running",
-                    "progress": entry.get("progress", {}),
-                    "started_at": entry.get("started_at", 0),
-                })
-        return {"active": active}
-
-    @router.get("/api/research/status/{session_id}")
-    async def research_status(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        status = research_handler.get_status(session_id)
-        if status is None:
-            raise HTTPException(404, "No research found for this session")
-        return status
-
-    @router.post("/api/research/cancel/{session_id}")
-    async def research_cancel(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        cancelled = research_handler.cancel_research(session_id)
-        return {"cancelled": cancelled}
-
-    @router.post("/api/research/result/{session_id}")
-    async def research_result(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research result available")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
-
-    def _assert_owns_research(session_id: str, user: str) -> None:
-        """404-not-403 ownership gate for a research session's on-disk JSON.
-        Use BEFORE returning any data or mutating the file."""
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
-        except Exception:
-            raise HTTPException(404, "Research not found")
-        if owner != user:
-            raise HTTPException(404, "Research not found")
-
-    @router.get("/api/research/report/{session_id}")
-    async def research_report(session_id: str, request: Request):
-        """Serve the visual HTML report for a completed research session."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        logger.info(f"Visual report requested for session {session_id}")
-        try:
-            html_content = research_handler.get_report_html(session_id)
-        except Exception as e:
-            logger.error(f"Visual report generation error: {e}", exc_info=True)
-            raise HTTPException(500, f"Report generation failed: {e}")
-        if html_content is None:
-            logger.warning(f"No report data found for session {session_id}")
-            raise HTTPException(404, "No visual report available for this session")
-        return HTMLResponse(content=html_content)
-
-    class HideImageRequest(BaseModel):
-        url: str
-
-    @router.post("/api/research/{session_id}/hide-image")
-    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
-        """Mark an image URL as hidden for this research's visual report.
-        Persisted to the research JSON so subsequent /report renders skip it."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.hide_image(session_id, body.url)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.post("/api/research/{session_id}/unhide-images")
-    async def research_unhide_images(session_id: str, request: Request):
-        """Clear the hidden-images list for a research session."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.unhide_all_images(session_id)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.get("/api/research/library")
-    async def research_library(
-        request: Request,
-        search: Optional[str] = Query(None),
-        sort: str = Query("recent"),
-        limit: int = Query(50),
-        archived: bool = Query(False),
-    ):
-        user = _require_user(request)
-        """List all completed research for the Library panel."""
-        data_dir = Path(DEEP_RESEARCH_DIR)
-        items = []
-        for p in data_dir.glob("*.json"):
-            try:
-                d = json.loads(p.read_text(encoding="utf-8"))
-                # SECURITY: only show research belonging to this user. Legacy
-                # JSONs without an `owner` field are hidden — auth was the only
-                # gate before, so every user saw every other user's reports.
-                if d.get("owner") != user:
-                    continue
-                # Archived view shows ONLY archived reports; default hides them.
-                if bool(d.get("archived")) != archived:
-                    continue
-                query = d.get("query", "")
-                if search and search.lower() not in query.lower():
-                    continue
-                sources = d.get("sources", [])
-                items.append({
-                    "id": p.stem,
-                    "query": query,
-                    "category": d.get("category") or "",
-                    "source_count": len(sources),
-                    "status": d.get("status", "done"),
-                    "duration": d.get("stats", {}).get("Duration", ""),
-                    "rounds": d.get("stats", {}).get("Rounds", ""),
-                    "started_at": d.get("started_at", 0),
-                    "completed_at": d.get("completed_at", 0),
-                    "archived": bool(d.get("archived")),
-                })
-            except Exception:
-                continue
-
-        # Sort
-        if sort == "recent":
-            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
-        elif sort == "oldest":
-            items.sort(key=lambda x: x["completed_at"] or 0)
-        elif sort == "most-messages":
-            items.sort(key=lambda x: x["source_count"], reverse=True)
-        elif sort == "alpha":
-            items.sort(key=lambda x: x["query"].lower())
-
-        return {"research": items[:limit], "total": len(items)}
-
-    @router.get("/api/research/detail/{session_id}")
-    async def research_detail(session_id: str, request: Request):
-        """Return the full JSON for a single research result — sources,
-        summary, stats — used by the Library preview panel."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-        except Exception as e:
-            raise HTTPException(500, f"Failed to read research: {e}")
-        # SECURITY: 404 (not 403) so we don't leak that the report exists.
-        if data.get("owner") != user:
-            raise HTTPException(404, "Research not found")
-        return data
-
-    @router.post("/api/research/{session_id}/archive")
-    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
-        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-            if data.get("owner") != user:
-                raise HTTPException(404, "Research not found")
-            data["archived"] = bool(archived)
-            path.write_text(json.dumps(data), encoding="utf-8")
-        except HTTPException:
-            raise
-        except Exception as e:
-            raise HTTPException(500, f"Failed to update research: {e}")
-        return {"ok": True, "id": session_id, "archived": bool(archived)}
-
-    @router.delete("/api/research/{session_id}")
-    async def research_delete(session_id: str, request: Request):
-        """Delete a research result from disk."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        data_dir = Path(DEEP_RESEARCH_DIR)
-        json_path = data_dir / f"{session_id}.json"
-        deleted = False
-        if json_path.exists():
-            # SECURITY: verify ownership before letting the caller delete it.
-            try:
-                data = json.loads(json_path.read_text(encoding="utf-8"))
-                if data.get("owner") != user:
-                    raise HTTPException(404, "Research not found")
-            except HTTPException:
-                raise
-            except Exception:
-                raise HTTPException(404, "Research not found")
-            json_path.unlink()
-            deleted = True
-        return {"deleted": deleted}
-
-    # ------------------------------------------------------------------
-    # Panel endpoints — launch research without a chat session
-    # ------------------------------------------------------------------
-
-    class ResearchStartRequest(BaseModel):
-        query: str
-        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
-        max_rounds: int = Field(default=0, ge=0, le=20)
-        search_provider: Optional[str] = None
-        endpoint_id: Optional[str] = None
-        model: Optional[str] = None
-        max_time: int = Field(default=300, ge=60, le=1800)
-        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
-        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
-        category: Optional[str] = None
-
-    @router.post("/api/research/start")
-    async def research_start(body: ResearchStartRequest, request: Request):
-        """Launch a research job from the dedicated panel."""
-        from src.auth_helpers import require_privilege
-        user = require_privilege(request, "can_use_research")
-        if user == INTERNAL_TOOL_USER:
-            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
-            if tool_owner and tool_owner not in RESERVED_USERNAMES:
-                auth_mgr = getattr(request.app.state, "auth_manager", None)
-                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
-                    try:
-                        privs = auth_mgr.get_privileges(tool_owner) or {}
-                        if not privs.get("can_use_research", True):
-                            raise HTTPException(403, f"Your account is not allowed to can use research.")
-                    except HTTPException:
-                        raise
-                    except Exception:
-                        pass
-                user = tool_owner
-        session_id = f"rp-{uuid.uuid4().hex[:12]}"
-
-        if body.endpoint_id:
-            from src.database import SessionLocal
-            db = SessionLocal()
-            try:
-                # Owner-scoped: never resolve another user's private endpoint
-                # (and its decrypted api_key / internal base_url). A scoped miss
-                # reads as 404 so the endpoint's existence isn't revealed.
-                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
-                if not ep:
-                    raise HTTPException(404, "Endpoint not found or disabled")
-                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
-                if not resolved:
-                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
-                ep_url, ep_model, ep_headers = resolved
-            finally:
-                db.close()
-        else:
-            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
-            # When neither research nor utility is configured, use the user's
-            # configured DEFAULT model (default_endpoint_id/default_model) rather
-            # than arbitrarily grabbing the first enabled endpoint's first model
-            # (which surfaced gpt-3.5). "Default" should mean the default model.
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
-            if not ep_url:
-                from src.database import SessionLocal
-                db = SessionLocal()
-                try:
-                    # Owner-scoped first-enabled fallback: the caller's own rows
-                    # + legacy null-owner shared rows only — never borrow another
-                    # user's private endpoint/api_key. Same fix as the
-                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
-                    ep = _owned_enabled_endpoint(db, user)
-                    if ep:
-                        resolved = _resolve_endpoint_runtime(ep, owner=user)
-                        if resolved:
-                            ep_url, ep_model, ep_headers = resolved
-                finally:
-                    db.close()
-            if not ep_url:
-                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
-            if body.model:
-                ep_model = body.model
-
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
-        research_handler.start_research(
-            session_id=session_id,
-            query=body.query,
-            llm_endpoint=ep_url,
-            llm_model=ep_model,
-            max_time=body.max_time,
-            llm_headers=ep_headers,
-            max_rounds=effective_max_rounds,
-            search_provider=body.search_provider or None,
-            category=body.category or None,
-            extraction_timeout=body.extraction_timeout,
-            extraction_concurrency=body.extraction_concurrency,
-            owner=user,
-        )
-        return {"session_id": session_id, "status": "running", "query": body.query}
-
-    @router.get("/api/research/stream/{session_id}")
-    async def research_stream(session_id: str, request: Request):
-        """SSE stream of research progress events."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        async def _generate():
-            last_progress = None
-            while True:
-                status = research_handler.get_status(session_id)
-                if status is None:
-                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
-                    return
-                st = status.get("status", "")
-                progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
-                if st != "running":
-                    final = {'status': st, 'final': True}
-                    task = research_handler._active_tasks.get(session_id, {})
-                    if st == "error" and task.get("result"):
-                        final['error'] = str(task["result"])[:500]
-                    yield f"data: {json.dumps(final)}\n\n"
-                    return
-                await asyncio.sleep(1.5)
-
-        return StreamingResponse(
-            _generate(),
-            media_type="text/event-stream",
-            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
-        )
-
-    @router.post("/api/research/result-peek/{session_id}")
-    async def research_result_peek(session_id: str, request: Request):
-        """Get research result without clearing it (for panel use)."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-            if p.exists():
-                d = json.loads(p.read_text(encoding="utf-8"))
-                return {
-                    "result": d.get("result", ""),
-                    "sources": d.get("sources", []),
-                    "raw_findings": d.get("raw_findings", []),
-                    "category": d.get("category") or "",
-                }
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
-
-    @router.post("/api/research/spinoff/{session_id}")
-    async def research_spinoff(session_id: str, request: Request):
-        """Create a new chat session pre-seeded with this research as context.
-
-        Reads the persisted research result + sources for `session_id`, creates
-        a fresh session (inheriting endpoint/model/headers from the source
-        session if available, otherwise from the resolved chat endpoint), and
-        injects a single system message containing the report and sources so
-        the user can ask follow-up questions in a clean conversation.
-        """
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        # SECURITY: gate on ownership before reading the persisted research —
-        # otherwise any authenticated user could spin off (and thereby read)
-        # another user's report by guessing its session ID. Mirrors every other
-        # endpoint in this file (see result_peek above).
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        if session_manager is None:
-            raise HTTPException(500, "session_manager not configured")
-
-        # Load research data — prefer in-memory result, fall back to disk
-        result = research_handler.get_result(session_id)
-        sources = research_handler.get_sources(session_id) or []
-        query = ""
-
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if path.exists():
-            try:
-                disk = json.loads(path.read_text(encoding="utf-8"))
-                if not result:
-                    result = disk.get("result")
-                if not sources:
-                    sources = disk.get("sources", []) or []
-                query = disk.get("query", "") or ""
-            except Exception as e:
-                logger.warning(f"Could not read research JSON for spinoff: {e}")
-
-        if not result:
-            raise HTTPException(404, "No research result available for this session")
-
-        # Inherit endpoint/model/headers from the source session when possible.
-        # For panel-launched research (rp-* IDs), there is no chat session, so
-        # fall back through the same chain as /api/research/start: research →
-        # utility → first enabled endpoint in the DB.
-        ep_url, ep_model, ep_headers = "", "", {}
-        try:
-            src_sess = session_manager.get_session(session_id)
-            ep_url = src_sess.endpoint_url or ""
-            ep_model = src_sess.model or ""
-            ep_headers = dict(src_sess.headers or {})
-        except KeyError:
-            pass
-
-        def _merge(r_url, r_model, r_headers):
-            nonlocal ep_url, ep_model, ep_headers
-            if not ep_url and r_url:
-                ep_url = r_url
-            if not ep_model and r_model:
-                ep_model = r_model
-            if not ep_headers and r_headers:
-                ep_headers = dict(r_headers)
-
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("chat", owner=user))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("research", owner=user))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("utility", owner=user))
-        if not ep_url or not ep_model:
-            # Last resort: this user's enabled endpoint, plus legacy shared rows.
-            from src.database import SessionLocal
-            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
-            db = SessionLocal()
-            try:
-                ep = _owned_enabled_endpoint(db, user)
-                if ep:
-                    base = normalize_base(ep.base_url)
-                    fallback_url = build_chat_url(base)
-                    fallback_headers = build_headers(ep.api_key, base)
-                    fallback_model = ""
-                    if ep.cached_models:
-                        try:
-                            models = json.loads(ep.cached_models)
-                            if models:
-                                fallback_model = _first_chat_model(models)
-                        except Exception:
-                            pass
-                    _merge(fallback_url, fallback_model, fallback_headers)
-            finally:
-                db.close()
-
-        if not ep_url or not ep_model:
-            raise HTTPException(400, "No endpoint configured — add one in Settings first")
-
-        # Create new session
-        new_sid = str(uuid.uuid4())
-
-        title_query = (query or "research").strip()
-        if len(title_query) > 60:
-            title_query = title_query[:57] + "…"
-        new_name = f"Follow-up: {title_query}"
-
-        new_sess = session_manager.create_session(
-            session_id=new_sid,
-            name=new_name,
-            endpoint_url=ep_url,
-            model=ep_model,
-            rag=False,
-            owner=user,
-        )
-        if ep_headers:
-            new_sess.headers = ep_headers
-            session_manager.save_sessions()
-        try:
-            from src.event_bus import fire_event
-            fire_event("session_created", user)
-        except Exception:
-            logger.debug("session_created event dispatch failed", exc_info=True)
-
-        # Build the priming system message — report only, no sources injected.
-        # The user can open the visual report for source details; keeping sources
-        # out of the chat context saves tokens and avoids the AI fabricating
-        # citations.
-        date_str = datetime.utcnow().strftime("%Y-%m-%d")
-        primer = (
-            f"[Research context — {date_str}]\n\n"
-            f"The user previously ran a deep research investigation. Use the "
-            f"report below as your primary knowledge base when answering "
-            f"follow-up questions. If the user asks something not covered, "
-            f"say so plainly rather than guessing.\n\n"
-            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
-            f"=== REPORT ===\n{result}"
-        )
-
-        from core.models import ChatMessage
-        new_sess.add_message(ChatMessage(
-            role="system",
-            content=primer,
-            metadata={"research_spinoff_from": session_id},
-        ))
-        session_manager.save_sessions()
-
-        return {
-            "session_id": new_sid,
-            "name": new_name,
-            "source_count": len(sources),
-        }
-
-    return router
+_sys.modules[__name__] = _canonical

routes/session_routes.py

@@ -162,7 +162,7 @@ def _persist_session_headers(session_id: str, headers: dict | None) -> None:
         db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
         if db_session:
             db_session.headers = headers or {}
-            db_session.updated_at = datetime.utcnow()
+            db_session.updated_at = utcnow_naive()
             db.commit()
     except Exception:
         db.rollback()
@@ -223,8 +223,8 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         # purge exists only to catch ghosts the frontend missed (tab close,
         # crash). Only clean up rows old enough to be definitely orphaned.
         try:
-            from datetime import datetime as _dt, timedelta as _td
-            _cutoff = _dt.utcnow() - _td(minutes=10)
+            from datetime import timedelta as _td
+            _cutoff = utcnow_naive() - _td(minutes=10)
             _purge_db = SessionLocal()
             try:
                 from core.database import ChatMessage as _DbMsg
@@ -470,7 +470,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == sid).first()
                 if db_session:
                     db_session.folder = folder if folder else None
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
                     result["folder"] = folder if folder else None
             finally:
@@ -517,7 +517,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                     db_session.model = model
                     db_session.endpoint_url = endpoint_url
                     db_session.headers = session.headers or {}
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
             finally:
                 db.close()
@@ -646,7 +646,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == sid).first()
                 if db_session:
                     db_session.archived = True
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
                     
                     # Update in memory if it exists
@@ -680,7 +680,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             if not db_session:
                 raise HTTPException(404, f"Session {sid} not found")
             db_session.archived = False
-            db_session.updated_at = datetime.utcnow()
+            db_session.updated_at = utcnow_naive()
             db.commit()
             # Reload into session manager so it appears in the active list
             try:
@@ -890,7 +890,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
                 if db_session:
                     db_session.is_important = important
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
 
                     # Update in memory if it exists
@@ -979,7 +979,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             metadata={
                 "compacted": True,
                 "summarized_count": len(older),
-                "timestamp": datetime.utcnow().isoformat(),
+                "timestamp": utcnow_naive().isoformat(),
             },
         )
         new_history = [summary_msg] + recent
@@ -1256,7 +1256,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db_session_q.first()
                 if db_session:
                     db_session.folder = folder_name
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     updated += 1
             db.commit()
         except Exception as e:

routes/shell_routes.py

@@ -16,6 +16,11 @@ from pathlib import Path
 from typing import Dict, Any
 from core.platform_compat import IS_APPLE_SILICON, which_tool
 from core.middleware import INTERNAL_TOOL_USER
+from src.host_docker_access import (
+    HOST_DOCKER_ACCESS_HINT,
+    host_docker_access_enabled as _host_docker_access_enabled,
+    running_in_container as _running_in_container,
+)
 from src.optional_deps import prepare_optional_dependency_import
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
@@ -103,32 +108,17 @@ logger = logging.getLogger(__name__)
 PTY_SUPPORTED = pty is not None and fcntl is not None and hasattr(os, "setsid")
 
 
-DOCKER_IN_CONTAINER_HINT = (
-    "Not available inside the Odysseus container by design. The image ships no "
-    "docker CLI and no host socket is mounted. Run Docker-backed launches on a "
-    "remote server, where docker is checked over SSH. Mounting /var/run/docker.sock "
-    "into the container would grant it host-root access, so only do that if you "
-    "accept that risk."
-)
-
-
-def _running_in_container(dockerenv_path="/.dockerenv", cgroup_path="/proc/1/cgroup"):
-    if os.path.exists(dockerenv_path):
-        return True
-    try:
-        with open(cgroup_path, "r", encoding="utf-8") as fh:
-            contents = fh.read()
-    except OSError:
-        return False
-    return any(token in contents for token in ("docker", "containerd", "kubepods"))
+DOCKER_IN_CONTAINER_HINT = HOST_DOCKER_ACCESS_HINT
 
 
 DockerRowStatus = namedtuple("DockerRowStatus", ["applicable", "install_hint"])
 PackageUpdateStatus = namedtuple("PackageUpdateStatus", ["available", "note"])
 
 
-def _docker_row_status(*, on_remote, in_container, installed, default_hint):
-    local_docker_unavailable = not on_remote and in_container and not installed
+def _docker_row_status(
+    *, on_remote, in_container, installed, default_hint, host_docker_access=False
+):
+    local_docker_unavailable = not on_remote and in_container and not host_docker_access
     if local_docker_unavailable:
         return DockerRowStatus(applicable=False, install_hint=DOCKER_IN_CONTAINER_HINT)
     return DockerRowStatus(applicable=True, install_hint=default_hint)
@@ -331,6 +321,9 @@ def add_user_install_bins_to_path():
         candidates.append(os.path.join(site.USER_BASE, 'bin'))
     except Exception:
         pass
+    candidates.append(os.path.expanduser('~/bin'))
+    candidates.append(os.path.expanduser('~/llama.cpp/build/bin'))
+    candidates.append(os.path.expanduser('~/llama.cpp/build-vulkan/bin'))
     candidates.append(os.path.expanduser('~/.local/bin'))
     parts = os.environ.get('PATH', '').split(os.pathsep) if os.environ.get('PATH') else []
     changed = False
@@ -962,12 +955,84 @@ def setup_shell_routes() -> APIRouter:
 
         return StreamingResponse(generate(), media_type="text/event-stream")
 
+    def _os_id_from_release(text: str) -> str:
+        """Map /etc/os-release contents to a canonical family for our matrix."""
+        if not text:
+            return ""
+        ids = []
+        for line in text.splitlines():
+            line = line.strip()
+            if line.startswith("ID=") or line.startswith("ID_LIKE="):
+                ids += line.split("=", 1)[1].strip().strip('"').split()
+        ids = [i.lower() for i in ids]
+        if any(x in ids for x in ("debian", "ubuntu", "linuxmint", "pop", "elementary")):
+            return "debian"
+        if any(x in ids for x in ("arch", "manjaro", "endeavouros", "cachyos", "garuda")):
+            return "arch"
+        if any(x in ids for x in ("fedora", "rhel", "centos", "rocky", "almalinux", "ol")):
+            return "fedora"
+        if "alpine" in ids:
+            return "alpine"
+        if any(x in ids for x in ("suse", "opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles")):
+            return "suse"
+        return ""
+
+    # Matrix lookup keyed on (os_family, backend) → (pkg_mgr_cmd_template, pkg_list_per_dep).
+    # Each `system_prereqs` name resolves to a list of OS-specific package
+    # names that get joined into the final `sudo apt install -y …` etc.
+    # command. Backend-specific extras (CUDA toolkit, ROCm, Vulkan headers)
+    # are added only when the detected backend needs them.
+    _PKG_NAMES = {
+        # canonical-name → {os_id: [actual_pkg_names_on_this_os]}
+        "cmake":           {"debian": ["cmake"], "arch": ["cmake"], "fedora": ["cmake"], "alpine": ["cmake"], "suse": ["cmake"], "macos": ["cmake"]},
+        "build-essential": {"debian": ["build-essential"], "arch": ["base-devel"], "fedora": ["gcc", "gcc-c++", "make"], "alpine": ["build-base"], "suse": ["gcc-c++", "make"], "macos": []},
+        "g++":             {"debian": ["g++"], "arch": ["gcc"], "fedora": ["gcc-c++"], "alpine": ["g++"], "suse": ["gcc-c++"], "macos": []},
+        "gcc":             {"debian": ["gcc"], "arch": ["gcc"], "fedora": ["gcc"], "alpine": ["gcc"], "suse": ["gcc"], "macos": []},
+        "make":            {"debian": ["make"], "arch": ["make"], "fedora": ["make"], "alpine": ["make"], "suse": ["make"], "macos": []},
+        "git":             {"debian": ["git"], "arch": ["git"], "fedora": ["git"], "alpine": ["git"], "suse": ["git"], "macos": ["git"]},
+        "tmux":            {"debian": ["tmux"], "arch": ["tmux"], "fedora": ["tmux"], "alpine": ["tmux"], "suse": ["tmux"], "macos": ["tmux"]},
+    }
+    _BACKEND_EXTRAS = {
+        "cuda":   {"debian": ["nvidia-cuda-toolkit"], "arch": ["cuda"], "fedora": ["cuda-toolkit"], "alpine": [], "suse": ["cuda"], "macos": []},
+        "rocm":   {"debian": ["rocm-dev"], "arch": ["rocm-hip-sdk"], "fedora": ["rocm-devel"], "alpine": [], "suse": ["rocm-dev"], "macos": []},
+        "vulkan": {"debian": ["libvulkan-dev", "vulkan-tools"], "arch": ["vulkan-headers", "vulkan-tools"], "fedora": ["vulkan-headers", "vulkan-tools"], "alpine": ["vulkan-loader-dev", "vulkan-tools"], "suse": ["vulkan-devel", "vulkan-tools"], "macos": []},
+    }
+    _PKG_MGR = {
+        "debian": "sudo apt install -y {pkgs}",
+        "arch":   "sudo pacman -S --needed {pkgs}",
+        "fedora": "sudo dnf install -y {pkgs}",
+        "alpine": "sudo apk add {pkgs}",
+        "suse":   "sudo zypper install -n {pkgs}",
+        "macos":  "brew install {pkgs}",
+    }
+
+    def _install_cmd_for_target(os_id: str, backend: str, missing: list[str]) -> str:
+        """Build a single OS+backend-aware install command for the missing prereqs."""
+        if not os_id or os_id not in _PKG_MGR:
+            return ""
+        pkgs: list[str] = []
+        seen: set[str] = set()
+        for m in missing:
+            for p in _PKG_NAMES.get(m, {}).get(os_id, []):
+                if p not in seen:
+                    pkgs.append(p); seen.add(p)
+        # Add backend-specific extras only when the build would actually
+        # consume them (a CUDA toolkit isn't useful on a Vulkan box).
+        backend = (backend or "").lower()
+        for p in _BACKEND_EXTRAS.get(backend, {}).get(os_id, []):
+            if p not in seen:
+                pkgs.append(p); seen.add(p)
+        if not pkgs:
+            return ""
+        return _PKG_MGR[os_id].format(pkgs=" ".join(pkgs))
+
     @router.get("/api/cookbook/packages")
     async def list_packages(
         request: Request,
         host: str | None = None,
         ssh_port: str | None = None,
         venv: str | None = None,
+        backend: str | None = None,
     ):
         """Check which optional packages are installed.
 
@@ -988,8 +1053,19 @@ def setup_shell_routes() -> APIRouter:
         importlib.invalidate_caches()
         try:
             user_site = site.getusersitepackages()
-            if user_site and os.path.isdir(user_site) and user_site not in sys.path:
-                sys.path.append(user_site)
+            if user_site and os.path.isdir(user_site):
+                # Use addsitedir(), NOT a bare sys.path.append(). When a package
+                # is `pip install --user`'d at runtime (Cookbook → Install) the
+                # long-lived server process started before the user-site existed,
+                # so site never processed it — including its `.pth` hooks. On
+                # Python 3.12+ `distutils` is gone from stdlib and is only
+                # restored by setuptools' `distutils-precedence.pth`, which ships
+                # in user-site. basicsr (a realesrgan dep) does `import distutils`
+                # at import time, so a plain append left the package importable
+                # but `import distutils` failing → realesrgan probed as
+                # not-installed until a full process restart. addsitedir() replays
+                # the `.pth` files so the shim is active.
+                site.addsitedir(user_site)
         except Exception:
             pass
         if ssh_port and str(ssh_port).strip() not in ("", "22"):
@@ -1016,6 +1092,12 @@ def setup_shell_routes() -> APIRouter:
                 "kind": "system",
                 "install_hint": "Install Docker on the selected server and allow this user to run docker.",
             },
+            # Note: cmake / gcc / git are not separate dependency rows —
+            # they're declared as `system_prereqs` on llama_cpp (and any
+            # other engine that compiles from source) so they appear as
+            # an inline status note on that engine's row instead of
+            # cluttering the panel with raw OS package names that aren't
+            # meaningful product-level dependencies on their own.
             # ── LLM ── installs on GPU servers for model serving/downloading
             {
                 "name": "hf_transfer",
@@ -1027,9 +1109,16 @@ def setup_shell_routes() -> APIRouter:
             {
                 "name": "llama_cpp",
                 "pip": "llama-cpp-python[server]",
-                "desc": "Serve GGUF models via llama.cpp",
+                "desc": "Great for single-GPU or CPU inference with GGUF models",
                 "category": "LLM",
                 "target": "remote",
+                # Build-toolchain prereqs. Cookbook's launch bootstrap
+                # compiles llama-server from source when no prebuilt
+                # binary is present; without these the build aborts
+                # with `cmake: command not found`. Surfaced inline on
+                # this row so the user doesn't have to chase three
+                # separate OS-package rows.
+                "system_prereqs": ["cmake", "g++", "git"],
             },
             {
                 "name": "sglang",
@@ -1041,7 +1130,7 @@ def setup_shell_routes() -> APIRouter:
             {
                 "name": "vllm",
                 "pip": "vllm",
-                "desc": "High-throughput LLM serving engine",
+                "desc": "Great for high-throughput multi-GPU inference",
                 "category": "LLM",
                 "target": "remote",
             },
@@ -1060,7 +1149,7 @@ def setup_shell_routes() -> APIRouter:
             {
                 "name": "diffusers",
                 "pip": "diffusers[torch]",
-                "desc": "Image generation pipelines (SD, Flux) with PyTorch",
+                "desc": "Image generation/editing pipelines (SD, Flux) with PyTorch",
                 "category": "Image",
                 "target": "remote",
             },
@@ -1104,6 +1193,7 @@ def setup_shell_routes() -> APIRouter:
         # venv over SSH so a remote `pip install` actually reflects here.
         remote_status: dict = {}
         remote_details: dict = {}
+        remote_probe_error = ""
         remote_names = [
             p["name"]
             for p in packages
@@ -1142,16 +1232,56 @@ def setup_shell_routes() -> APIRouter:
                         break
             except ValueError as e:
                 raise HTTPException(400, str(e))
-            except Exception:
+            except Exception as e:
                 remote_status = {}
-        if host and remote_system_names:
+                remote_probe_error = f"SSH package probe failed: {str(e)[:160]}"
+            if "llama_cpp" in remote_names:
+                try:
+                    inner = (
+                        'export PATH="$HOME/.local/bin:$HOME/bin:'
+                        '$HOME/llama.cpp/build/bin:$HOME/llama.cpp/build-vulkan/bin:$PATH"; '
+                        "command -v llama-server 2>/dev/null || true"
+                    )
+                    argv = _ssh_base_argv(host, ssh_port) + [inner]
+                    proc = await asyncio.create_subprocess_exec(
+                        *argv,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.PIPE,
+                    )
+                    out, _err = await asyncio.wait_for(proc.communicate(), timeout=8)
+                    llama_server_path = out.decode("utf-8", errors="replace").strip().splitlines()
+                    llama_server_path = llama_server_path[-1].strip() if llama_server_path else ""
+                    if llama_server_path:
+                        remote_status["llama_cpp"] = True
+                        probe = remote_details.setdefault("llama_cpp", {})
+                        if isinstance(probe, dict):
+                            probe.setdefault("binaries", {})["llama-server"] = llama_server_path
+                except Exception as e:
+                    if not remote_probe_error:
+                        remote_probe_error = f"SSH llama-server probe failed: {str(e)[:160]}"
+                    pass
+        # Union of system_names + every package's system_prereqs. Probing
+        # the prereqs alongside the main system deps in a single SSH call
+        # avoids a second round-trip per Cookbook → Dependencies refresh.
+        prereq_names: set[str] = set()
+        for p in packages:
+            for pr in p.get("system_prereqs") or []:
+                prereq_names.add(str(pr))
+        all_system_names = list(set(remote_system_names) | prereq_names)
+        # Detect the target's OS family + read /etc/os-release in the same
+        # SSH round-trip as the prereq probe — used downstream to render a
+        # single OS-specific install command per row instead of dumping
+        # every distro's syntax onto the user.
+        target_os_id: str = ""
+        if host and all_system_names:
             try:
                 checks = []
-                for name in remote_system_names:
+                for name in all_system_names:
                     qn = shlex.quote(name)
                     checks.append(
                         f"if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
                     )
+                checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || true")
                 inner = " ; ".join(checks)
                 argv = _ssh_base_argv(host, ssh_port) + [inner]
                 proc = await asyncio.create_subprocess_exec(
@@ -1161,20 +1291,45 @@ def setup_shell_routes() -> APIRouter:
                 )
                 out, _err = await asyncio.wait_for(proc.communicate(), timeout=12)
                 txt = out.decode("utf-8", errors="replace").strip()
+                _section, _osrel_lines = "probe", []
                 for line in txt.splitlines():
+                    if line.strip() == "---OSREL---":
+                        _section = "osrel"; continue
+                    if _section == "osrel":
+                        _osrel_lines.append(line)
+                        continue
                     name, sep, value = line.strip().partition("=")
-                    if sep and name in remote_system_names:
+                    if sep and name in all_system_names:
                         remote_status[name] = value == "1"
+                target_os_id = _os_id_from_release("\n".join(_osrel_lines))
             except ValueError as e:
                 raise HTTPException(400, str(e))
-            except Exception:
+            except Exception as e:
+                if not remote_probe_error:
+                    remote_probe_error = f"SSH system probe failed: {str(e)[:160]}"
                 pass
+        elif not host:
+            # Local target — probe in-process so the inline install command
+            # still appears in the dep panel when the cookbook container
+            # itself is the selected server.
+            try:
+                with open("/etc/os-release", encoding="utf-8") as f:
+                    target_os_id = _os_id_from_release(f.read())
+            except Exception:
+                target_os_id = ""
+            if sys.platform == "darwin":
+                target_os_id = "macos"
 
         for pkg in packages:
             on_remote = bool(host and pkg.get("target") == "remote")
             probe = None
             if on_remote:
-                pkg["installed"] = bool(remote_status.get(pkg["name"], False))
+                if remote_probe_error and pkg["name"] not in remote_status:
+                    pkg["installed"] = None
+                    pkg["probe_error"] = remote_probe_error
+                    pkg["status_note"] = remote_probe_error
+                else:
+                    pkg["installed"] = bool(remote_status.get(pkg["name"], False))
                 probe = remote_details.get(pkg["name"])
                 if isinstance(probe, dict):
                     pkg["details"] = probe
@@ -1223,13 +1378,116 @@ def setup_shell_routes() -> APIRouter:
                     pkg["installed"] = False
                 except importlib_metadata.PackageNotFoundError:
                     pkg["installed"] = False
-                except Exception:
+                except (Exception, SystemExit):
                     # Installed but crashes on import — e.g. a CUDA build of
                     # llama-cpp-python raising FileNotFoundError when the CUDA
-                    # toolkit dir is absent. One broken optional package must not
-                    # 500 the entire packages panel; report it as not usable.
+                    # toolkit dir is absent, or rembg calling sys.exit(1) when no
+                    # onnxruntime backend can be loaded. SystemExit is a
+                    # BaseException, not Exception, so without catching it here a
+                    # single sys.exit-on-import package escapes and takes down the
+                    # whole packages panel / worker (the panel hangs forever). One
+                    # broken optional package must not 500 — or hang — the entire
+                    # panel; report it as not usable.
                     pkg["installed"] = False
 
+            # llama_cpp partial-state probe: when the package is installed
+            # but the wheel was built CPU-only AND the target has NVIDIA
+            # hardware, mark the row as partial (yellow/orange) with a
+            # one-click upgrade to the CUDA wheel. Without this the row
+            # reads "ready" green while inference runs at 3 tok/s on GPU
+            # silicon — actively misleading.
+            if pkg["name"] == "llama_cpp" and pkg.get("installed"):
+                _native_llama_server = bool(
+                    isinstance(probe, dict)
+                    and isinstance(probe.get("binaries"), dict)
+                    and probe["binaries"].get("llama-server")
+                )
+                _gpu_capable = False
+                _has_nvidia_target = False
+                if _native_llama_server:
+                    # Native llama-server is the launcher path Cookbook now
+                    # prefers. Do not mark this as a CPU-only Python wheel just
+                    # because llama-cpp-python is absent from the selected venv.
+                    _gpu_capable = True
+                elif on_remote and host:
+                    try:
+                        # Activate the configured venv FIRST so the probe
+                        # runs against the same python the launch script
+                        # would activate. Without this prefix, bare
+                        # `python3` was checked — which can disagree with
+                        # the venv's wheel (e.g. user-site has CUDA wheel
+                        # but venv has CPU-only), and the dep panel then
+                        # showed "ready" green while every launch fell to
+                        # CPU.
+                        _vp = _venv_activate_prefix(venv)
+                        probe = (
+                            f'{_vp}python3 -c "import llama_cpp; import sys; '
+                            'sys.exit(0 if llama_cpp.llama_supports_gpu_offload() else 1)" '
+                            '&& echo llama_cpp_gpu=1 || echo llama_cpp_gpu=0; '
+                            'command -v nvidia-smi >/dev/null 2>&1 '
+                            '&& nvidia-smi -L 2>/dev/null | grep -q "GPU " '
+                            '&& echo nvidia=1 || echo nvidia=0'
+                        )
+                        argv = _ssh_base_argv(host, ssh_port) + [probe]
+                        proc = await asyncio.create_subprocess_exec(
+                            *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+                        )
+                        out, _ = await asyncio.wait_for(proc.communicate(), timeout=8)
+                        txt = out.decode("utf-8", errors="replace")
+                        if "llama_cpp_gpu=1" in txt:
+                            _gpu_capable = True
+                        if "nvidia=1" in txt:
+                            _has_nvidia_target = True
+                    except Exception:
+                        pass
+                else:
+                    try:
+                        import llama_cpp as _lcp  # type: ignore
+                        _gpu_capable = bool(_lcp.llama_supports_gpu_offload())
+                    except Exception:
+                        _gpu_capable = False
+                    _has_nvidia_target = shutil.which("nvidia-smi") is not None
+                if (not _gpu_capable) and _has_nvidia_target:
+                    pkg["partial"] = True
+                    pkg["partial_reason"] = "Installed but CPU-only wheel — GPU detected on this target. Upgrade to a CUDA wheel for ~10× faster inference."
+                    pkg["partial_action"] = "reinstall_llama_cpp_cuda"
+            # Attach per-package system_prereqs status. We probed each
+            # prereq name above; surface "Missing build deps: …" ONLY
+            # when the package itself is not installed — if the package
+            # works (e.g. llama-cpp-python already imports cleanly), the
+            # build toolchain is irrelevant and surfacing it as a red
+            # flag confuses users ("ready" + "missing" on the same row).
+            _prereqs = list(pkg.get("system_prereqs") or [])
+            if _prereqs:
+                if on_remote:
+                    _pr_present = {n: bool(remote_status.get(n)) for n in _prereqs}
+                else:
+                    _pr_present = {n: shutil.which(n) is not None for n in _prereqs}
+                pkg["system_prereqs_status"] = _pr_present
+                _missing = [n for n, ok in _pr_present.items() if not ok]
+                # Suppress the "missing build deps" hint when the package
+                # itself is installed — build deps are only relevant if
+                # the user would need to recompile from source.
+                if pkg.get("installed"):
+                    _missing = []
+                if _missing:
+                    # Build a target-specific install command from the
+                    # (os_family, backend) matrix when we know both. Fall
+                    # back to the multi-distro hint only when the target's
+                    # OS can't be classified (e.g. ssh probe failed).
+                    _resolved_os = target_os_id or "debian"  # safest default
+                    _cmd = _install_cmd_for_target(_resolved_os, backend or "", _missing)
+                    if _cmd and target_os_id:
+                        _hint = "Missing build deps for this target: " + ", ".join(_missing)
+                        pkg["install_cmd_for_target"] = _cmd
+                        pkg["install_cmd_os"] = target_os_id
+                        pkg["install_cmd_backend"] = (backend or "").lower()
+                    else:
+                        _hint = "Missing build deps: " + ", ".join(_missing) + ". Install via apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git."
+                    _existing_note = pkg.get("status_note") or ""
+                    pkg["status_note"] = (_existing_note + " — " + _hint) if _existing_note else _hint
+                    pkg["build_deps_missing"] = _missing
+
             if pkg.get("installed"):
                 update_status = _package_pip_update_status(pkg, probe)
                 pkg["pip_update_available"] = update_status.available
@@ -1242,6 +1500,9 @@ def setup_shell_routes() -> APIRouter:
                     in_container=_running_in_container() if not on_remote else False,
                     installed=pkg["installed"],
                     default_hint=pkg.get("install_hint"),
+                    host_docker_access=(
+                        _host_docker_access_enabled() if not on_remote else False
+                    ),
                 )
                 pkg["applicable"] = status.applicable
                 pkg["install_hint"] = status.install_hint
@@ -1289,6 +1550,102 @@ def setup_shell_routes() -> APIRouter:
             return {"ok": True, "output": stdout.decode()[-200:]}
         return {"ok": False, "error": stderr.decode()[-300:]}
 
+    @router.post("/api/cookbook/install-system-deps")
+    async def install_system_deps(request: Request):
+        """Install OS-level system packages (cmake/build-essential/git/tmux)
+        on a remote target or in the local container. Admin only.
+
+        Bounded by a per-package allowlist — anything outside the catalog
+        is rejected so the route can't be coerced into installing arbitrary
+        OS packages. Uses `sudo -n` (passwordless) so the call returns a
+        clear "needs sudo password" error instead of hanging when interactive
+        sudo is required.
+        """
+        _require_admin(request)
+        body = await request.json()
+        raw = body.get("packages") or []
+        host = (body.get("remote_host") or "").strip()
+        ssh_port = body.get("ssh_port")
+        # Names users can request — must match canonical names used in the
+        # deps catalog's `system_prereqs` field and on the System rows.
+        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make"}
+        pkgs = [str(p).strip() for p in raw if str(p).strip() in ALLOWED]
+        if not pkgs:
+            return {"ok": False, "error": "no installable packages requested (allowlist: " + ", ".join(sorted(ALLOWED)) + ")"}
+        # Re-map to the right package name per OS. apt/dpkg use the names
+        # as-is; pacman has base-devel for build-essential, etc.
+        def _apt(names): return list(names)
+        def _pacman(names):
+            return ["base-devel" if n == "build-essential" else n for n in names]
+        def _dnf(names):
+            out = []
+            for n in names:
+                if n == "build-essential": out += ["gcc", "gcc-c++", "make"]
+                elif n == "g++": out += ["gcc-c++"]
+                else: out.append(n)
+            return out
+        def _brew(names):
+            return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
+        # Build a single shell snippet that detects the package manager and
+        # runs the right install. Non-interactive sudo (-n) only — if sudo
+        # asks for a password the script reports it instead of hanging.
+        apt_pkgs = " ".join(shlex.quote(p) for p in _apt(pkgs))
+        pac_pkgs = " ".join(shlex.quote(p) for p in _pacman(pkgs))
+        dnf_pkgs = " ".join(shlex.quote(p) for p in _dnf(pkgs))
+        brew_pkgs = " ".join(shlex.quote(p) for p in _brew(pkgs))
+        # Error messages go to stderr (>&2) so the route's error field
+        # gets populated. Without the redirect, `echo "ERROR…"` on stdout
+        # left stderr empty and the frontend toast fell through to a
+        # bare "HTTP 200" instead of surfacing the real reason.
+        script = (
+            'set -e; '
+            'if ! sudo -n true 2>/dev/null; then '
+            '  echo "ERROR: passwordless sudo unavailable on this target. Run once: sudo apt install -y ' + " ".join(pkgs) + ' (or your distro equivalent: pacman -S, dnf install, brew install). After that, Cookbook can install the rest." >&2; exit 2; fi; '
+            'if command -v apt-get >/dev/null 2>&1; then '
+            f'  sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq && sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
+            'elif command -v pacman >/dev/null 2>&1; then '
+            f'  sudo -n pacman -Sy --needed --noconfirm {pac_pkgs}; '
+            'elif command -v dnf >/dev/null 2>&1; then '
+            f'  sudo -n dnf install -y {dnf_pkgs}; '
+            'elif command -v brew >/dev/null 2>&1; then '
+            f'  brew install {brew_pkgs}; '
+            'else '
+            '  echo "ERROR: no supported package manager (apt/pacman/dnf/brew) on this target." >&2; exit 3; fi'
+        )
+        try:
+            if host:
+                argv = _ssh_base_argv(host, ssh_port) + [script]
+            else:
+                argv = ["bash", "-lc", script]
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            out, err = await asyncio.wait_for(proc.communicate(), timeout=180)
+        except asyncio.TimeoutError:
+            return {"ok": False, "error": "Install timed out after 180s"}
+        ok = (proc.returncode == 0)
+        # Combine stderr + (last lines of stdout) into a single error
+        # blob when ok=False — some package managers print useful failure
+        # context to stdout, and a script that exits via `echo ...; exit N`
+        # without `>&2` would otherwise hand back an empty error string
+        # and force the frontend to show a bare "HTTP 200".
+        err_txt = err.decode("utf-8", errors="replace").strip()
+        out_txt = out.decode("utf-8", errors="replace").strip()
+        if not ok:
+            tail_out = out_txt[-500:] if out_txt else ""
+            combined = err_txt or tail_out or f"exit code {proc.returncode}"
+        else:
+            combined = None
+        return {
+            "ok": ok,
+            "exit_code": proc.returncode,
+            "output": out_txt[-1000:],
+            "error": combined,
+        }
+
     @router.post("/api/cookbook/rebuild-engine")
     async def rebuild_engine(request: Request):
         """Clear the cached llama.cpp build so the next serve recompiles.
@@ -1309,7 +1666,8 @@ def setup_shell_routes() -> APIRouter:
             return {"ok": False, "error": f"Unsupported engine: {engine}"}
         host = str(body.get("remote_host") or "").strip()
         ssh_port = body.get("ssh_port")
-        cmd = _llama_cpp_rebuild_cmd()
+        update_source = bool(body.get("update_source"))
+        cmd = _llama_cpp_rebuild_cmd(update_source=update_source)
         try:
             argv = (
                 (_ssh_base_argv(host, ssh_port) + [cmd])

routes/skills_routes.py

@@ -22,6 +22,16 @@ from core.middleware import require_admin
 
 logger = logging.getLogger(__name__)
 
+# Last-resort verdict extraction from a teacher/verifier model's prose (run when
+# JSON parsing fails). `["\'\s:]*` already consumes whitespace, so the original
+# trailing `\s*` made two adjacent \s-matching quantifiers that backtrack O(n^2)
+# on a `verdict` + whitespace flood in untrusted model output (CodeQL
+# py/polynomial-redos). Without it a single unbounded quantifier remains — the
+# matched text is identical, and the scan is linear.
+_VERDICT_PROSE_RE = re.compile(
+    r'verdict["\'\s:]*["\']?(pass|needs_work|fail|inconclusive)', re.I
+)
+
 
 class SkillAddRequest(BaseModel):
     # New schema (preferred)
@@ -196,7 +206,7 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
         # Last resort: pull the verdict keyword straight out of the prose so a
         # clearly-decided run isn't thrown away as "unparseable".
         if v not in _VERDICTS:
-            km = _re.search(r'verdict["\'\s:]*\s*["\']?(pass|needs_work|fail|inconclusive)', text, _re.I)
+            km = _VERDICT_PROSE_RE.search(text)
             if km:
                 v = km.group(1).lower()
                 if data is None:

routes/task_routes.py

@@ -594,6 +594,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         cache_tables = {
             "summarize_emails": ("email_summaries",),
             "draft_email_replies": ("email_ai_replies",),
+            "email_auto_translate": ("email_translations",),
             "extract_email_events": ("email_calendar_extractions",),
             "learn_sender_signatures": ("sender_signatures",),
             "check_email_urgency": ("email_tags", "email_urgency_alerts"),
@@ -893,10 +894,11 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         return {"ok": True, "message": "Task stopped"}
 
     @router.get("/runs/recent")
-    async def list_recent_runs(request: Request, limit: int = 50):
+    async def list_recent_runs(request: Request, limit: int = 50, max_result_chars: int = 6000):
         """Recent task runs across ALL tasks for this owner. Drives the Activity view."""
         user = _owner(request)
         limit = max(1, min(limit, 200))
+        max_result_chars = max(500, min(max_result_chars, 20000))
         db = SessionLocal()
         try:
             q = db.query(TaskRun, ScheduledTask).join(
@@ -930,10 +932,20 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 deduped.append((r, t))
                 if len(deduped) >= limit:
                     break
+
+            def _clip_run(r: TaskRun) -> dict:
+                d = _run_to_dict(r)
+                for key in ("result", "error"):
+                    val = d.get(key)
+                    if isinstance(val, str) and len(val) > max_result_chars:
+                        d[key] = val[:max_result_chars].rstrip() + "\n\n[Activity preview truncated]"
+                return d
+
             return {
+                "has_more": len(rows) > len(deduped),
                 "runs": [
                     {
-                        **_run_to_dict(r),
+                        **_clip_run(r),
                         "task_name": _display_task_name(t),
                         "task_type": t.task_type or "llm",
                         "action": t.action,

routes/upload_routes.py

@@ -3,11 +3,16 @@ import os
 import time
 import json
 import asyncio
-from fastapi import APIRouter, Request, File, UploadFile, HTTPException
-from typing import List
+import shutil
+import uuid
+from pathlib import Path
+from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
+from typing import List, Optional
 import logging
 from core.middleware import require_admin
+from core.database import SessionLocal, GalleryImage, Session as DbSession
 from src.auth_helpers import effective_user
+from src.constants import GENERATED_IMAGES_DIR
 from src.upload_handler import count_recent_uploads
 
 logger = logging.getLogger(__name__)
@@ -50,10 +55,90 @@ def setup_upload_routes(upload_handler):
             raise HTTPException(404, "File not found")
 
         raise HTTPException(404, "File not found")
+
+    def _valid_session_id_for_owner(db, session_id: str | None, owner: str | None) -> str | None:
+        if not session_id:
+            return None
+        sess = db.query(DbSession).filter(DbSession.id == session_id).first()
+        if not sess:
+            return None
+        if owner and sess.owner and sess.owner != owner:
+            return None
+        return session_id
+
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
+        """Make chat-uploaded images visible in Gallery without changing chat storage."""
+        is_image_file = getattr(upload_handler, "is_image_file", None)
+        if not callable(is_image_file):
+            return None
+        if not is_image_file(meta.get("name", ""), meta.get("mime", "")):
+            return None
+
+        source_path = meta.get("path")
+        if not source_path or not os.path.isfile(source_path):
+            return None
+
+        db = SessionLocal()
+        try:
+            file_hash = meta.get("hash")
+            if file_hash:
+                q = db.query(GalleryImage).filter(
+                    GalleryImage.file_hash == file_hash,
+                    GalleryImage.is_active == True,  # noqa: E712
+                )
+                if owner:
+                    q = q.filter(GalleryImage.owner == owner)
+                existing = q.first()
+                if existing:
+                    return existing.id
+
+            image_dir = Path(GENERATED_IMAGES_DIR)
+            image_dir.mkdir(parents=True, exist_ok=True)
+            ext = Path(meta.get("name") or source_path).suffix.lower()
+            if ext not in {".png", ".jpg", ".jpeg", ".webp", ".gif"}:
+                mime_ext = {
+                    "image/png": ".png",
+                    "image/jpeg": ".jpg",
+                    "image/jpg": ".jpg",
+                    "image/webp": ".webp",
+                    "image/gif": ".gif",
+                }.get(meta.get("mime", ""))
+                ext = mime_ext or ".png"
+            filename = f"{uuid.uuid4().hex[:12]}{ext}"
+            dest_path = image_dir / filename
+            shutil.copy2(source_path, dest_path)
+
+            image_id = str(uuid.uuid4())
+            db.add(GalleryImage(
+                id=image_id,
+                filename=filename,
+                prompt=meta.get("name") or "Chat upload",
+                model="chat-upload",
+                owner=owner,
+                session_id=_valid_session_id_for_owner(db, session_id, owner),
+                file_hash=file_hash,
+                width=meta.get("width"),
+                height=meta.get("height"),
+                file_size=meta.get("size"),
+            ))
+            db.commit()
+            return image_id
+        except Exception as e:
+            db.rollback()
+            logger.warning("Failed to add chat image upload to gallery: %s", e)
+            return None
+        finally:
+            db.close()
     
     @router.post("")
-    async def api_upload(request: Request, files: List[UploadFile] = File(...)):
+    async def api_upload(
+        request: Request,
+        files: List[UploadFile] = File(...),
+        session_id: Optional[str] = Form(None),
+    ):
         """Upload files with enhanced security and organization."""
+        if not isinstance(session_id, str):
+            session_id = None
         if not files:
             raise HTTPException(400, "No files uploaded")
             
@@ -78,8 +163,10 @@ def setup_upload_routes(upload_handler):
         
         for u in files:
             try:
-                meta = upload_handler.save_upload(u, client_ip, owner=effective_user(request))
-                out.append({
+                owner = effective_user(request)
+                meta = upload_handler.save_upload(u, client_ip, owner=owner)
+                gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
+                item = {
                     "id": meta["id"],
                     "name": meta["name"],
                     "mime": meta["mime"],
@@ -89,7 +176,10 @@ def setup_upload_routes(upload_handler):
                     "width": meta.get("width"),
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
-                })
+                }
+                if gallery_id:
+                    item["gallery_id"] = gallery_id
+                out.append(item)
             except HTTPException:
                 raise
             except Exception as e:
@@ -128,14 +218,13 @@ def setup_upload_routes(upload_handler):
         import mimetypes as _mt
         # Look up original filename and owner from uploads.json
         original_name = file_id
-        info = None
-        uploads_db = os.path.join(_upload_root(), "uploads.json")
-        if os.path.exists(uploads_db):
-            with open(uploads_db, encoding="utf-8") as f:
-                db = json.load(f)
-            info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
-            if info:
-                original_name = info.get("name", file_id)
+        # _load_upload_index() tolerates a missing/corrupt uploads.json (it falls
+        # back to the .bak sibling, then to {}), so a truncated DB degrades to
+        # "no metadata" instead of a 500 from an unhandled JSONDecodeError.
+        db = upload_handler._load_upload_index()
+        info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
+        if info:
+            original_name = info.get("name", file_id)
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
         current_user = effective_user(request)
@@ -181,19 +270,42 @@ def setup_upload_routes(upload_handler):
 
     def _load_upload_info(file_id: str):
         """Look up the uploads.json record for a file_id, with owner/auth checks."""
-        info = None
-        uploads_db = os.path.join(_upload_root(), "uploads.json")
-        if os.path.exists(uploads_db):
-            with open(uploads_db, encoding="utf-8") as f:
-                db = json.load(f)
-            info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
-        return info
+        # Corruption-tolerant load (see download_file): a bad uploads.json yields
+        # {} rather than raising JSONDecodeError out of the vision path.
+        db = upload_handler._load_upload_index()
+        return next((fi for fi in db.values() if fi.get("id") == file_id), None)
 
     def _vision_cache_path(file_id: str) -> str:
         cache_dir = os.path.join(_upload_root(), ".vision")
         os.makedirs(cache_dir, exist_ok=True)
         return os.path.join(cache_dir, file_id + ".txt")
 
+    def _sync_gallery_caption_for_upload(info: dict | None, owner: str | None, text: str) -> None:
+        """Copy upload OCR/vision text onto the promoted gallery image row."""
+        if not info:
+            return
+        file_hash = info.get("hash")
+        if not file_hash:
+            return
+        db = SessionLocal()
+        try:
+            q = db.query(GalleryImage).filter(
+                GalleryImage.file_hash == file_hash,
+                GalleryImage.is_active == True,  # noqa: E712
+            )
+            if owner:
+                q = q.filter(GalleryImage.owner == owner)
+            img = q.first()
+            if not img:
+                return
+            img.caption = (text or "").strip()
+            db.commit()
+        except Exception as e:
+            db.rollback()
+            logger.warning("Failed to sync OCR caption to gallery image: %s", e)
+        finally:
+            db.close()
+
     @router.get("/{file_id}/vision")
     async def get_vision_text(request: Request, file_id: str, force: int = 0):
         """Return the vision-model OCR/description for an uploaded image.
@@ -220,7 +332,9 @@ def setup_upload_routes(upload_handler):
         if not force and os.path.exists(cache_path):
             try:
                 with open(cache_path, encoding="utf-8") as f:
-                    return {"text": f.read(), "cached": True}
+                    cached_text = f.read()
+                _sync_gallery_caption_for_upload(info, file_owner or current_user, cached_text)
+                return {"text": cached_text, "cached": True}
             except Exception as e:
                 logger.warning(f"Vision cache read failed for {file_id}: {e}")
         from src.document_processor import analyze_image_with_vl
@@ -234,6 +348,7 @@ def setup_upload_routes(upload_handler):
                 f.write(text)
         except Exception as e:
             logger.warning(f"Vision cache write failed for {file_id}: {e}")
+        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"text": text, "cached": False}
 
     @router.put("/{file_id}/vision")
@@ -255,12 +370,16 @@ def setup_upload_routes(upload_handler):
             if file_owner != current_user and not auth_mgr.is_admin(current_user):
                 raise HTTPException(404, "File not found")
         _resolve_upload_path(file_id)
-        body = await request.json()
+        try:
+            body = await request.json()
+        except json.JSONDecodeError:
+            raise HTTPException(400, "Request body must be valid JSON")
         text = (body or {}).get("text", "")
         if not isinstance(text, str):
             raise HTTPException(400, "text must be a string")
         with open(_vision_cache_path(file_id), "w", encoding="utf-8") as f:
             f.write(text)
+        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"ok": True}
 
     async def periodic_rate_limit_cleanup():

routes/webhook_routes.py

@@ -345,8 +345,9 @@ def setup_webhook_routes(
                             resp = await client.get(models_url, headers=hdrs)
                             resp.raise_for_status()
                             data = resp.json()
-                            ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
-                            if not ids:
+                            items = data if isinstance(data, list) else (data.get("data") or [])
+                            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
+                            if not ids and isinstance(data, dict):
                                 ids = [
                                     m.get("name") or m.get("model")
                                     for m in (data.get("models") or [])

scripts/claim_ownerless.py

@@ -27,12 +27,18 @@ def claim_json_entries(entries, owner):
     return count
 
 
+def owner_arg(argv):
+    if len(argv) < 2 or not argv[1].strip():
+        return None
+    return argv[1].strip()
+
+
 def main():
-    if len(sys.argv) < 2:
+    owner = owner_arg(sys.argv)
+    if not owner:
         print("Usage: python scripts/claim_ownerless.py <username>")
         sys.exit(1)
 
-    owner = sys.argv[1]
     print(f"Claiming all ownerless data for: {owner}\n")
 
     # 1. Memories (JSON files)

scripts/odysseus-gallery

@@ -38,23 +38,27 @@ def _preview_text(value, limit: int = 200) -> str:
     return text[:limit]
 
 
+def _text_field(value) -> str:
+    return value if isinstance(value, str) else ""
+
+
 def _serialize_image(i: "GalleryImage") -> dict:
     return {
         "id": i.id,
-        "filename": i.filename,
+        "filename": _text_field(i.filename),
         "prompt": _preview_text(i.prompt),
-        "model": i.model or "",
-        "size": i.size or "",
-        "tags": i.tags or "",
+        "model": _text_field(i.model),
+        "size": _text_field(i.size),
+        "tags": _text_field(i.tags),
         "favorite": bool(i.favorite),
-        "album_id": i.album_id or "",
-        "session_id": i.session_id or "",
+        "album_id": _text_field(i.album_id),
+        "session_id": _text_field(i.session_id),
         "width": i.width,
         "height": i.height,
         "file_size": i.file_size,
         "taken_at": i.taken_at.isoformat() if i.taken_at else "",
-        "camera_make": i.camera_make or "",
-        "camera_model": i.camera_model or "",
+        "camera_make": _text_field(i.camera_make),
+        "camera_model": _text_field(i.camera_model),
         "created_at": i.created_at.isoformat() if i.created_at else "",
     }
 
@@ -93,11 +97,11 @@ def cmd_show(args):
         if not i:
             fail(f"no image with id {args.id!r}")
         out = _serialize_image(i)
-        out["prompt_full"] = i.prompt or ""
-        out["ai_tags"] = i.ai_tags or ""
+        out["prompt_full"] = _text_field(i.prompt)
+        out["ai_tags"] = _text_field(i.ai_tags)
         out["gps_lat"] = i.gps_lat or ""
         out["gps_lng"] = i.gps_lng or ""
-        out["file_hash"] = i.file_hash or ""
+        out["file_hash"] = _text_field(i.file_hash)
         emit(out, args)
     finally:
         db.close()

scripts/odysseus-mail

@@ -108,6 +108,8 @@ def _q(name: str) -> str:
 
 
 def _split_recipients(value: str) -> list[str]:
+    if not isinstance(value, str):
+        return []
     return [r.strip() for r in (value or "").split(",") if r.strip()]
 
 

scripts/odysseus-notes

@@ -36,7 +36,9 @@ def _load_items(raw) -> list:
         items = json.loads(raw)
     except (TypeError, json.JSONDecodeError):
         return []
-    return items if isinstance(items, list) else []
+    if not isinstance(items, list):
+        return []
+    return [item for item in items if isinstance(item, dict)]
 
 
 def _serialize(n: "Note") -> dict:

services/hwfit/data/hf_models.json

@@ -5113,8 +5113,8 @@
  {
   "name": "deepseek-ai/DeepSeek-V4-Flash",
   "provider": "deepseek-ai",
-  "parameter_count": "284B",
-  "parameters_raw": 284000000000,
+  "parameter_count": "158.1B",
+  "parameters_raw": 158069433298,
   "active_parameters": 13000000000,
   "is_moe": true,
   "min_ram_gb": 200.0,
@@ -5130,15 +5130,40 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 3542202,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 1882337,
+  "hf_likes": 1651,
+  "release_date": "2026-06-22"
+ },
+ {
+  "name": "deepseek-ai/DeepSeek-V4-Flash-DSpark",
+  "provider": "deepseek-ai",
+  "parameter_count": "165.3B",
+  "parameters_raw": 165265454782,
+  "active_parameters": 13000000000,
+  "is_moe": true,
+  "active_experts": 6,
+  "min_ram_gb": 170.0,
+  "recommended_ram_gb": 250.0,
+  "min_vram_gb": 165.0,
+  "quantization": "FP8-Mixed",
+  "context_length": 1000000,
+  "use_case": "General-purpose reasoning, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "deepseek_v4_moe",
+  "hf_downloads": 4446,
+  "hf_likes": 107,
+  "release_date": "2026-06-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Flash-Base",
   "provider": "deepseek-ai",
-  "parameter_count": "284B",
-  "parameters_raw": 284000000000,
+  "parameter_count": "292.0B",
+  "parameters_raw": 292021347282,
   "active_parameters": 13000000000,
   "is_moe": true,
   "min_ram_gb": 290.0,
@@ -5153,15 +5178,15 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 0,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 76030,
+  "hf_likes": 256,
+  "release_date": "2026-04-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Pro",
   "provider": "deepseek-ai",
-  "parameter_count": "1.6T",
-  "parameters_raw": 1600000000000,
+  "parameter_count": "861.6B",
+  "parameters_raw": 861608274846,
   "active_parameters": 49000000000,
   "is_moe": true,
   "min_ram_gb": 1100.0,
@@ -5177,15 +5202,40 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 0,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 1154610,
+  "hf_likes": 5118,
+  "release_date": "2026-06-22"
+ },
+ {
+  "name": "deepseek-ai/DeepSeek-V4-Pro-DSpark",
+  "provider": "deepseek-ai",
+  "parameter_count": "889.5B",
+  "parameters_raw": 889484881098,
+  "active_parameters": 49000000000,
+  "is_moe": true,
+  "active_experts": 6,
+  "min_ram_gb": 900.0,
+  "recommended_ram_gb": 1250.0,
+  "min_vram_gb": 890.0,
+  "quantization": "FP8-Mixed",
+  "context_length": 1000000,
+  "use_case": "Flagship reasoning, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "deepseek_v4_moe",
+  "hf_downloads": 6939,
+  "hf_likes": 241,
+  "release_date": "2026-06-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Pro-Base",
   "provider": "deepseek-ai",
   "parameter_count": "1.6T",
-  "parameters_raw": 1600000000000,
+  "parameters_raw": 1600790440862,
   "active_parameters": 49000000000,
   "is_moe": true,
   "min_ram_gb": 1700.0,
@@ -5200,9 +5250,9 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 0,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 25387,
+  "hf_likes": 305,
+  "release_date": "2026-04-27"
  },
  {
   "name": "deepseek-ai/deepseek-coder-6.7b-base",
@@ -13308,6 +13358,106 @@
   "_discovered": true,
   "gguf_sources": []
  },
+ {
+  "name": "zai-org/GLM-5.2",
+  "provider": "zai-org",
+  "parameter_count": "753.3B",
+  "parameters_raw": 753329940480,
+  "min_ram_gb": 1510.0,
+  "recommended_ram_gb": 1800.0,
+  "min_vram_gb": 1510.0,
+  "quantization": "BF16",
+  "context_length": 1048576,
+  "use_case": "General purpose reasoning, coding, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "glm_moe_dsa",
+  "hf_downloads": 142547,
+  "hf_likes": 2996,
+  "release_date": "2026-06-23",
+  "is_moe": true,
+  "active_experts": 8,
+  "gguf_sources": [
+   {
+    "repo": "unsloth/GLM-5.2-GGUF",
+    "provider": "unsloth",
+    "file": "UD-Q4_K_M/*.gguf",
+    "quant": "Q4_K_M"
+   }
+  ]
+ },
+ {
+  "name": "zai-org/GLM-5.2-FP8",
+  "provider": "zai-org",
+  "parameter_count": "753.4B",
+  "parameters_raw": 753375793584,
+  "min_ram_gb": 760.0,
+  "recommended_ram_gb": 900.0,
+  "min_vram_gb": 760.0,
+  "quantization": "FP8",
+  "context_length": 1048576,
+  "use_case": "General purpose reasoning, coding, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "glm_moe_dsa",
+  "hf_downloads": 884226,
+  "hf_likes": 182,
+  "release_date": "2026-06-23",
+  "is_moe": true,
+  "active_experts": 8,
+  "gguf_sources": [
+   {
+    "repo": "unsloth/GLM-5.2-GGUF",
+    "provider": "unsloth",
+    "file": "UD-Q4_K_M/*.gguf",
+    "quant": "Q4_K_M"
+   }
+  ]
+ },
+ {
+  "name": "unsloth/GLM-5.2-GGUF",
+  "provider": "unsloth",
+  "parameter_count": "753.9B",
+  "parameters_raw": 753864139008,
+  "min_ram_gb": 452.0,
+  "recommended_ram_gb": 620.0,
+  "min_vram_gb": 452.0,
+  "quantization": "Q4_K_M",
+  "context_length": 1048576,
+  "use_case": "General purpose reasoning, coding, long-context (GGUF)",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "glm-dsa",
+  "hf_downloads": 180394,
+  "hf_likes": 474,
+  "release_date": "2026-06-23",
+  "is_moe": true,
+  "active_experts": 8,
+  "is_gguf": true,
+  "gguf_sources": [
+   {
+    "repo": "unsloth/GLM-5.2-GGUF",
+    "provider": "unsloth",
+    "file": "UD-Q4_K_M/*.gguf",
+    "quant": "Q4_K_M"
+   }
+  ]
+ },
  {
   "name": "cyankiwi/Qwen3.5-35B-A3B-AWQ-4bit",
   "provider": "cyankiwi",
@@ -14059,6 +14209,138 @@
    "vision"
   ]
  },
+ {
+  "name": "google/gemma-4-12B-it",
+  "provider": "Google",
+  "parameter_count": "12.0B",
+  "parameters_raw": 12000000000,
+  "min_ram_gb": 8.5,
+  "recommended_ram_gb": 11.0,
+  "min_vram_gb": 7.5,
+  "quantization": "Q4_K_M",
+  "context_length": 131072,
+  "use_case": "General purpose, multimodal; unsloth/gemma-4-12B-it-GGUF Dynamic variants reduce VRAM from ~7.5 GB to ~5.5 GB",
+  "is_moe": false,
+  "num_experts": null,
+  "active_experts": null,
+  "active_parameters": null,
+  "architecture": "gemma4",
+  "pipeline_tag": "image-text-to-text",
+  "release_date": "2026-04-01",
+  "gguf_sources": [
+   {
+    "repo": "unsloth/gemma-4-12B-it-GGUF",
+    "provider": "unsloth"
+   }
+  ],
+  "capabilities": [
+   "vision"
+  ]
+ },
+ {
+  "name": "google/gemma-4-12B-it-qat-int4",
+  "provider": "Google",
+  "parameter_count": "12.0B",
+  "parameters_raw": 12000000000,
+  "min_ram_gb": 8.0,
+  "recommended_ram_gb": 9.5,
+  "min_vram_gb": 6.5,
+  "quantization": "QAT-INT4",
+  "context_length": 131072,
+  "use_case": "General purpose, multimodal (QAT quantization-aware training — higher quality than post-train INT4; vLLM native; no GGUF)",
+  "is_moe": false,
+  "num_experts": null,
+  "active_experts": null,
+  "active_parameters": null,
+  "architecture": "gemma4",
+  "pipeline_tag": "image-text-to-text",
+  "release_date": "2026-04-01",
+  "gguf_sources": [],
+  "capabilities": [
+   "vision"
+  ]
+ },
+ {
+  "name": "google/gemma-4-12B-it-qat-int8",
+  "provider": "Google",
+  "parameter_count": "12.0B",
+  "parameters_raw": 12000000000,
+  "min_ram_gb": 15.0,
+  "recommended_ram_gb": 20.0,
+  "min_vram_gb": 13.5,
+  "quantization": "QAT-INT8",
+  "context_length": 131072,
+  "use_case": "General purpose, multimodal (QAT INT8 — highest quality, 2x VRAM of QAT-INT4; vLLM native; no GGUF)",
+  "is_moe": false,
+  "num_experts": null,
+  "active_experts": null,
+  "active_parameters": null,
+  "architecture": "gemma4",
+  "pipeline_tag": "image-text-to-text",
+  "release_date": "2026-04-01",
+  "gguf_sources": [],
+  "capabilities": [
+   "vision"
+  ]
+ },
+ {
+  "name": "google/gemma-4-12B-it-qat-q4_0-gguf",
+  "provider": "Google",
+  "parameter_count": "12.0B",
+  "parameters_raw": 12000000000,
+  "min_ram_gb": 8.5,
+  "recommended_ram_gb": 11.0,
+  "min_vram_gb": 7.5,
+  "quantization": "QAT-INT4",
+  "context_length": 262144,
+  "use_case": "General purpose, multimodal (vision + audio); official Google QAT int4 GGUF — near-bf16 quality at int4 size, served on llama.cpp/Ollama with CPU offload",
+  "is_moe": false,
+  "num_experts": null,
+  "active_experts": null,
+  "active_parameters": null,
+  "architecture": "gemma4",
+  "pipeline_tag": "image-text-to-text",
+  "release_date": "2026-04-01",
+  "gguf_sources": [
+   {
+    "repo": "google/gemma-4-12B-it-qat-q4_0-gguf",
+    "provider": "Google",
+    "file": "gemma-4-12b-it-qat-q4_0.gguf"
+   }
+  ],
+  "capabilities": [
+   "vision",
+   "audio"
+  ]
+ },
+ {
+  "name": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf",
+  "provider": "Google",
+  "parameter_count": "25.2B",
+  "parameters_raw": 25200000000,
+  "min_ram_gb": 14.4,
+  "recommended_ram_gb": 18.0,
+  "min_vram_gb": 14.4,
+  "quantization": "QAT-INT4",
+  "context_length": 262144,
+  "use_case": "High-throughput, multimodal MoE (3.8B active); official Google QAT int4 GGUF — near-bf16 quality at int4 size, served on llama.cpp with CPU offload",
+  "is_moe": true,
+  "num_experts": null,
+  "active_experts": null,
+  "active_parameters": 3800000000,
+  "architecture": "gemma4",
+  "pipeline_tag": "image-text-to-text",
+  "release_date": "2026-04-01",
+  "gguf_sources": [
+   {
+    "repo": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf",
+    "provider": "Google"
+   }
+  ],
+  "capabilities": [
+   "vision"
+  ]
+ },
  {
   "name": "google/gemma-4-31B-it",
   "provider": "Google",
@@ -18823,6 +19105,54 @@
   "active_experts": 8,
   "active_parameters": 13600000000
  },
+ {
+  "name": "MiniMaxAI/MiniMax-M3",
+  "provider": "MiniMaxAI",
+  "parameter_count": "427.0B",
+  "parameters_raw": 427040140160,
+  "min_ram_gb": 855.0,
+  "recommended_ram_gb": 1025.0,
+  "min_vram_gb": 855.0,
+  "quantization": "BF16",
+  "context_length": 1000000,
+  "use_case": "Vision, chat, coding, agentic tool use",
+  "capabilities": [
+   "vision",
+   "tool_use",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "image-text-to-text",
+  "architecture": "minimax_m3_vl",
+  "hf_downloads": 192311,
+  "hf_likes": 1267,
+  "release_date": "2026-06-23",
+  "is_moe": true
+ },
+ {
+  "name": "MiniMaxAI/MiniMax-M3-MXFP8",
+  "provider": "MiniMaxAI",
+  "parameter_count": "440.3B",
+  "parameters_raw": 440279845760,
+  "min_ram_gb": 445.0,
+  "recommended_ram_gb": 560.0,
+  "min_vram_gb": 445.0,
+  "quantization": "MXFP8",
+  "context_length": 1000000,
+  "use_case": "Vision, chat, coding, agentic tool use",
+  "capabilities": [
+   "vision",
+   "tool_use",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "image-text-to-text",
+  "architecture": "minimax_m3_vl",
+  "hf_downloads": 572278,
+  "hf_likes": 43,
+  "release_date": "2026-06-15",
+  "is_moe": true
+ },
  {
   "name": "bullerwins/MiniMax-M2.7-REAP-172B-fp8",
   "provider": "bullerwins",

services/hwfit/fit.py

@@ -9,7 +9,7 @@ from services.hwfit.models import (
 GPU_BANDWIDTH = {
     "5090": 1792, "5080": 960, "5070 ti": 896, "5070": 672, "5060 ti": 448, "5060": 256,
     "4090": 1008, "4080 super": 736, "4080": 717, "4070 ti super": 672, "4070 ti": 504, "4070 super": 504, "4070": 504, "4060 ti": 288, "4060": 272,
-    "3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360,
+    "3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360, "3050 ti": 192, "3050": 224,
     "2080 ti": 616, "2080 super": 496, "2080": 448, "2070 super": 448, "2070": 448, "2060 super": 448, "2060": 336,
     "1660 ti": 288, "1660 super": 336, "1660": 192, "1650 super": 192, "1650": 128,
     "h100 sxm": 3350, "h100": 2039, "h200": 4800, "a100 sxm": 2039, "a100": 1555,

services/hwfit/hardware.py

@@ -282,7 +282,17 @@ def _detect_amd():
             "gpus": cards,
             "gpu_groups": groups,
             "homogeneous": len(groups) <= 1,
-            "backend": "rocm",
+            # Pick the actual runtime label: ROCm/HIP only when its
+            # toolchain is installed, otherwise Vulkan if vulkaninfo is
+            # present (mesa RADV works fine on RDNA/CDNA when ROCm
+            # packages are absent — see Strix Halo where ROCm support
+            # is still backporting). Reporting "rocm" on a Vulkan-only
+            # host misleads downstream env-var pinning
+            # (HIP_VISIBLE_DEVICES is a no-op there).
+            "backend": (
+                "rocm" if (_run(["which", "rocminfo"]) or _run(["which", "hipconfig"]))
+                else ("vulkan" if _run(["which", "vulkaninfo"]) else "rocm")
+            ),
             "unified_memory": is_apu,
             # AMD ISA/family so downstream can tell datacenter Instinct (CDNA,
             # where vLLM/SGLang run AWQ/GPTQ reliably) from consumer Radeon
@@ -528,6 +538,32 @@ def _powershell_exe():
     path so we don't depend on a particular PATH ordering."""
     return shutil.which("pwsh") or shutil.which("powershell") or "powershell"
 
+def _powershell_encoded_for_ssh(script: str):
+    """Run a PowerShell script on a remote Windows host over SSH.
+
+    Nested quotes in powershell -Command break when passed through Windows
+    OpenSSH's cmd wrapper; -EncodedCommand avoids that.
+    """
+    import base64
+    encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii")
+    return _run(f"powershell -NoProfile -EncodedCommand {encoded}")
+
+
+def _probe_remote_platform():
+    """Best-effort OS detection over SSH when the caller didn't pass platform."""
+    out = _run("echo %OS%")
+    if out and "Windows_NT" in out:
+        return "windows"
+    uname = (_run(["uname", "-s"]) or "").strip().lower()
+    if uname == "darwin":
+        # Mac uses the linux detection path (_detect_apple_silicon over SSH).
+        return "linux"
+    if uname == "linux":
+        out = _run("test -d /data/data/com.termux && echo termux || echo linux")
+        if out and "termux" in out:
+            return "termux"
+    return "linux"
+
 
 def _detect_windows():
     """Detect Windows hardware via PowerShell/WMI.
@@ -590,9 +626,8 @@ def _detect_windows():
     """
     )
     if _remote_host:
-        # Remote: ship a single command string over SSH. The remote shell parses
-        # the quoting; PowerShell on the far side runs the -Command payload.
-        out = _run(f'powershell -Command "{ps_cmd}"')
+        # Remote: use -EncodedCommand so OpenSSH/cmd quoting does not break the script.
+        out = _powershell_encoded_for_ssh(ps_cmd.strip())
     else:
         # Local: pass a LIST argv straight to subprocess so the OS hands ps_cmd
         # to PowerShell verbatim — no fragile string-level quote escaping. Prefer
@@ -763,6 +798,13 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
     """
     global _remote_host, _remote_port, _remote_platform
 
+    if host and not platform:
+        _remote_host = host
+        _remote_port = ssh_port or None
+        platform = _probe_remote_platform()
+        _remote_host = None
+        _remote_port = None
+
     cache_key = _cache_key(host, ssh_port, platform)
     now = time.time()
     if not fresh and cache_key in _cache_by_host:
@@ -783,8 +825,8 @@ def detect_system(host="", ssh_port="", platform="", fresh=False):
             _remote_platform = None
             _cache_by_host[cache_key] = (now, result)
             return result
-        # If Windows detection failed, return error
-        result = {"error": f"Cannot connect to {host}", "host": host}
+        # SSH may work while the PowerShell hardware probe still fails.
+        result = {"error": f"Windows hardware probe failed for {host}", "host": host}
         _remote_host = None
         _remote_platform = None
         _cache_by_host[cache_key] = (now, result)

services/hwfit/models.py

@@ -12,6 +12,7 @@ QUANT_BPP = {
     "Q4_K_M": 0.58, "Q4_0": 0.58, "Q3_K_M": 0.48, "Q2_K": 0.37,
     "AWQ-4bit": 0.50, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.50, "GPTQ-Int8": 1.0,
+    "QAT-INT4": 0.50, "QAT-INT8": 1.0,
     "mlx-4bit": 0.55, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
     # DeepSeek-V4-style mixed: MoE experts in FP4 (bulk), attention + non-
     # expert dense in FP8, embeddings/LM head in BF16. By weight count the
@@ -30,6 +31,7 @@ QUANT_SPEED_MULT = {
     "Q4_K_M": 1.15, "Q4_0": 1.15, "Q3_K_M": 1.25, "Q2_K": 1.35,
     "AWQ-4bit": 1.2, "AWQ-8bit": 0.85,
     "GPTQ-Int4": 1.2, "GPTQ-Int8": 0.85,
+    "QAT-INT4": 1.15, "QAT-INT8": 0.85,
     "mlx-4bit": 1.15, "mlx-8bit": 0.85, "mlx-6bit": 1.0,
     "FP4-MoE-Mixed": 1.10,  # slightly slower than pure FP4 because of mixed-dtype dispatch
     "FP8-Mixed": 0.85,
@@ -47,6 +49,10 @@ QUANT_QUALITY_PENALTY = {
     # penalty so FP8 wins when both fit. AWQ-4bit stays heavier.
     "AWQ": -1.0, "AWQ-4bit": -4.0, "AWQ-8bit": -1.0,
     "GPTQ": -1.0, "GPTQ-Int4": -4.0, "GPTQ-Int8": -1.0,
+    # Quantization-aware training recovers most of the int4 quality loss, so a
+    # QAT-INT4 build lands far closer to bf16 than a post-training Q4/INT4
+    # (Google reports near-bf16 quality). Penalize it lightly, not like Q4_K_M.
+    "QAT-INT4": -1.0, "QAT-INT8": 0.0,
     "mlx-4bit": -4.0, "mlx-8bit": -0.5, "mlx-6bit": -1.5,
     # DeepSeek-V4 mixed: only MoE experts at FP4 (the rest is FP8/BF16),
     # so the realized quality is much closer to FP8 than to pure FP4 —
@@ -63,6 +69,7 @@ QUANT_BYTES_PER_PARAM = {
     "Q4_K_M": 0.5, "Q4_0": 0.5, "Q3_K_M": 0.375, "Q2_K": 0.25,
     "AWQ-4bit": 0.5, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.5, "GPTQ-Int8": 1.0,
+    "QAT-INT4": 0.5, "QAT-INT8": 1.0,
     "mlx-4bit": 0.5, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
     "FP4-MoE-Mixed": 0.55,
     "FP8-Mixed": 1.0,
@@ -74,6 +81,7 @@ PREQUANTIZED_PREFIXES = (
     "AWQ-", "GPTQ-", "mlx-", "FP8", "FP4", "NVFP4", "MXFP4", "NF4",
     "INT4", "INT8", "W4A16", "W8A8", "W8A16",
     "FP4-MoE-Mixed", "FP8-Mixed",
+    "QAT-",
 )
 
 

services/hwfit/profiles.py

@@ -103,6 +103,9 @@ def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=Non
     in the actual serving knobs (n_cpu_moe, KV-cache type, context). serve_quant
     is the file's quant label (e.g. "Q4_K_M") just for display.
     """
+    if not isinstance(system, dict) or not isinstance(model, dict):
+        return []
+
     vram = float(system.get("gpu_vram_gb") or 0)
     if vram <= 0:
         return []

services/search/content.py

@@ -8,11 +8,13 @@ import os
 import re
 import logging
 import socket
+import ssl
 from datetime import datetime, timedelta
-from typing import List
+from typing import Iterable, List, cast
 from urllib.parse import urljoin, urlparse
 
 import httpx
+import httpcore
 from bs4 import BeautifulSoup
 
 from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
@@ -91,6 +93,148 @@ def _public_http_url(url: str) -> bool:
         return False
 
 
+def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname:
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    host = (parsed.hostname or "").strip().lower()
+    if host in ("localhost", "metadata", "metadata.google.internal"):
+        raise httpx.RequestError(f"Blocked non-public hostname: {host}")
+    try:
+        ip = ipaddress.ip_address(host)
+        if _is_private_address(ip):
+            raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
+        return [ip]
+    except httpx.RequestError:
+        raise
+    except ValueError:
+        pass
+    addrs = _resolve_hostname_ips(host)
+    if not addrs or any(_is_private_address(a) for a in addrs):
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    return addrs
+
+
+class _PinnedBackend(httpcore.NetworkBackend):
+    """Network backend that connects to a pre-resolved IP.
+
+    httpcore derives the TLS SNI and the ``Host`` header from the URL's
+    origin, not from the host argument passed to ``connect_tcp``. So
+    routing the TCP connect to a resolved IP while leaving the URL
+    untouched keeps SNI / vhost behaviour correct and closes the
+    DNS-rebinding TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress):
+        self._ip = str(ip)
+        self._real = httpcore.SyncBackend()
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options=None,
+    ):
+        return self._real.connect_tcp(
+            self._ip, port, timeout, local_address, socket_options
+        )
+
+    def connect_unix_socket(self, path, timeout=None, socket_options=None):
+        return self._real.connect_unix_socket(path, timeout, socket_options)
+
+    def sleep(self, seconds: float) -> None:
+        return self._real.sleep(seconds)
+
+
+# Map httpcore exception classes to their httpx equivalents. Built
+# once at import time from the public exception classes; avoids any
+# import of httpx's private transport machinery. httpcore's
+# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
+# close and retry on its own) — we never expect to see it surface to
+# a transport caller, so it has no httpx counterpart here.
+_HTTPCORE_TO_HTTPX_EXC = {
+    httpcore.ConnectError: httpx.ConnectError,
+    httpcore.ConnectTimeout: httpx.ConnectTimeout,
+    httpcore.LocalProtocolError: httpx.LocalProtocolError,
+    httpcore.NetworkError: httpx.NetworkError,
+    httpcore.PoolTimeout: httpx.PoolTimeout,
+    httpcore.ProtocolError: httpx.ProtocolError,
+    httpcore.ProxyError: httpx.ProxyError,
+    httpcore.ReadError: httpx.ReadError,
+    httpcore.ReadTimeout: httpx.ReadTimeout,
+    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
+    httpcore.TimeoutException: httpx.TimeoutException,
+    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
+    httpcore.WriteError: httpx.WriteError,
+    httpcore.WriteTimeout: httpx.WriteTimeout,
+}
+
+
+class _PinnedTransport(httpx.BaseTransport):
+    """Transport that pins every TCP connect to a pre-resolved IP.
+
+    Uses only the public ``httpcore`` and ``httpx`` APIs — no
+    subclassing of ``httpx.HTTPTransport``, no reads of private
+    ``httpcore.ConnectionPool`` attributes, no imports from
+    ``httpx private transport internals``. The URL is passed through unchanged so SNI
+    / vhost work as if httpx had been given the hostname directly;
+    only the TCP destination is pinned, closing the DNS-rebinding
+    TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
+        self._pool = httpcore.ConnectionPool(
+            ssl_context=ssl.create_default_context(),
+            http1=True,
+            http2=http2,
+            network_backend=_PinnedBackend(ip),
+        )
+
+    def __enter__(self):
+        self._pool.__enter__()
+        return self
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
+        self._pool.__exit__(exc_type, exc_value, traceback)
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        httpcore_req = httpcore.Request(
+            method=request.method,
+            url=httpcore.URL(
+                scheme=request.url.raw_scheme,
+                host=request.url.raw_host,
+                port=request.url.port,
+                target=request.url.raw_path,
+            ),
+            headers=request.headers.raw,
+            content=request.stream,
+            extensions=request.extensions,
+        )
+        try:
+            httpcore_resp = self._pool.handle_request(httpcore_req)
+            # Eager materialisation matches the original
+            # ``response.text`` usage in fetch_webpage_content. The
+            # sync pool's stream is a plain Iterable[bytes] despite
+            # the httpcore type hint unioning the async variant.
+            content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
+        except Exception as exc:
+            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
+            if mapped is not None:
+                raise mapped(str(exc)) from exc
+            raise
+
+        return httpx.Response(
+            status_code=httpcore_resp.status,
+            headers=httpcore_resp.headers,
+            content=content,
+            extensions=httpcore_resp.extensions,
+        )
+
+    def close(self) -> None:
+        self._pool.close()
+
 class BodyTooLargeError(Exception):
     """The server declared a body larger than the hard fetch ceiling."""
 
@@ -141,78 +285,78 @@ class _CappedFetch:
 
 def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
                     max_bytes: int = None) -> "_CappedFetch":
-    """Capped streaming GET with SSRF-guarded manual redirects.
+    """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
 
-    The body is streamed and buffering stops at ``max_bytes`` (default: the
-    soft cap), so an oversized resource cannot be pulled into memory or the
-    content cache in full. When Content-Length already declares a body over
-    the hard ceiling, the fetch is refused before any body bytes are read.
+    Each hop is resolved once, validated as public, and then the actual TCP
+    connection is pinned to that resolved IP. The request URL is left unchanged
+    so Host and TLS SNI keep the original hostname.
     """
     cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
     current = url
     for _ in range(max_redirects + 1):
-        if not _public_http_url(current):
-            raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
+        ips = _resolve_public_ips(current)
+
         # Force identity transfer-encoding. With gzip/deflate the wire bytes
-        # (and Content-Length) can be a small fraction of the decoded body, so
-        # a tiny compressed response could pass the hard-cap preflight and then
-        # expand past the ceiling in a single decoded chunk before the streamed
-        # cap below can slice it. Identity makes Content-Length the true body
-        # size and keeps each streamed chunk bounded by the network read.
+        # and Content-Length can be a small fraction of the decoded body, so a
+        # tiny compressed response could pass the hard-cap preflight and then
+        # expand past the ceiling in one decoded chunk before the streamed cap
+        # below can slice it.
         req_headers = dict(headers or {})
         req_headers["Accept-Encoding"] = "identity"
-        with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
-                          follow_redirects=False) as response:
-            if response.status_code in (301, 302, 303, 307, 308):
-                location = response.headers.get("location")
-                if not location:
-                    return _CappedFetch(response.status_code, response.headers, b"",
-                                        False, None, response.encoding, str(response.url))
-                current = urljoin(str(response.url), location)
-                continue
 
-            # A server can ignore the identity request and still return a
-            # compressed body; httpx.iter_bytes would then decode it, and a tiny
-            # gzip can balloon into one decoded chunk far past the cap before we
-            # slice. Refuse a compressed Content-Encoding so the streamed cap
-            # stays a real memory bound (Content-Length is the compressed wire
-            # length here, so the preflight and size metadata are unreliable too).
-            enc = (response.headers.get("content-encoding") or "").strip().lower()
-            if enc and enc != "identity":
-                raise httpx.RequestError(
-                    f"Refusing compressed response (Content-Encoding: {enc}) after "
-                    "requesting identity: cannot bound decoded body size",
-                    request=httpx.Request("GET", current),
-                )
+        with httpx.Client(
+            headers=req_headers,
+            timeout=timeout,
+            follow_redirects=False,
+            transport=_PinnedTransport(ips[0]),
+        ) as client:
+            with client.stream("GET", current) as response:
+                if response.status_code in (301, 302, 303, 307, 308):
+                    location = response.headers.get("location")
+                    if not location:
+                        return _CappedFetch(response.status_code, response.headers, b"",
+                                            False, None, response.encoding, str(response.url))
+                    current = urljoin(str(response.url), location)
+                    continue
 
-            declared = None
-            raw_len = response.headers.get("content-length")
-            if raw_len and raw_len.isdigit():
-                declared = int(raw_len)
-            # Refuse before buffering anything when the server already tells
-            # us the body exceeds the absolute ceiling (Content-Length is wire
-            # bytes; the decompressed body can only be larger).
-            if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
-                raise BodyTooLargeError(current, declared)
+                # A server can ignore the identity request and still return a
+                # compressed body; httpx.iter_bytes would then decode it, and a
+                # tiny gzip can balloon into one decoded chunk far past the cap.
+                # Refuse compressed Content-Encoding so the streamed cap stays
+                # a real memory bound.
+                enc = (response.headers.get("content-encoding") or "").strip().lower()
+                if enc and enc != "identity":
+                    raise httpx.RequestError(
+                        f"Refusing compressed response (Content-Encoding: {enc}) after "
+                        "requesting identity: cannot bound decoded body size",
+                        request=httpx.Request("GET", current),
+                    )
+
+                declared = None
+                raw_len = response.headers.get("content-length")
+                if raw_len and raw_len.isdigit():
+                    declared = int(raw_len)
+
+                if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
+                    raise BodyTooLargeError(current, declared)
+
+                chunks = []
+                read = 0
+                truncated = False
+                for chunk in response.iter_bytes():
+                    read += len(chunk)
+                    if read > cap:
+                        keep = cap - (read - len(chunk))
+                        if keep > 0:
+                            chunks.append(chunk[:keep])
+                        truncated = True
+                        break
+                    chunks.append(chunk)
+
+                return _CappedFetch(response.status_code, response.headers,
+                                    b"".join(chunks), truncated, declared,
+                                    response.encoding, str(response.url))
 
-            chunks = []
-            read = 0
-            truncated = False
-            # We requested identity above, so iter_bytes yields the raw body in
-            # network-read-sized chunks (no decompression expansion); the cap
-            # therefore bounds what we actually buffer.
-            for chunk in response.iter_bytes():
-                read += len(chunk)
-                if read > cap:
-                    keep = cap - (read - len(chunk))
-                    if keep > 0:
-                        chunks.append(chunk[:keep])
-                    truncated = True
-                    break
-                chunks.append(chunk)
-            return _CappedFetch(response.status_code, response.headers,
-                                b"".join(chunks), truncated, declared,
-                                response.encoding, str(response.url))
     raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
 
 # PDF extraction (optional dependency)

setup.py

@@ -239,6 +239,15 @@ def check_arch():
 def main():
     print("\n=== Odysseus Setup ===\n")
 
+    # Load .env so pre-seeded ODYSSEUS_ADMIN_USER / ODYSSEUS_ADMIN_PASSWORD (and
+    # other deployment vars) are honored on native installs, not just when they
+    # are exported in the shell. Mirrors app.py: encoding="utf-8-sig" tolerates a
+    # UTF-8 BOM in a Notepad-saved .env. load_dotenv does not override already
+    # exported OS env vars, so the existing precedence is preserved. python-dotenv
+    # is a hard dependency (requirements.txt) and is verified by check_deps below.
+    from dotenv import load_dotenv
+    load_dotenv(os.path.join(BASE_DIR, ".env"), encoding="utf-8-sig")
+
     # Fail fast with a clear message if the CPU architecture is wrong (Apple
     # Silicon under an x86/Rosetta Python) before importing anything native.
     check_arch()

src/agent_runs.py

@@ -174,8 +174,20 @@ async def subscribe(session_id: str) -> AsyncGenerator[str, None]:
             next_seq += 1
         if run.status != "running":
             return
+        heartbeat_idx = 0
         while True:
-            seq, ev = await q.get()
+            try:
+                seq, ev = await asyncio.wait_for(q.get(), timeout=10.0)
+            except asyncio.TimeoutError:
+                # Keep slow local models/proxies alive while they prefill before
+                # the first token. SSE comments are ignored by the UI but reset
+                # browser/proxy idle timers, which prevents "empty response"
+                # disconnects on llama.cpp first-token latencies of 30s+.
+                if run.status == "running":
+                    heartbeat_idx += 1
+                    yield f": heartbeat {heartbeat_idx}\n\n"
+                    continue
+                seq, ev = (None, None)
             if seq is None:            # end sentinel
                 while next_seq < len(run.buffer):   # flush any tail the sentinel raced
                     yield run.buffer[next_seq]

src/agent_tools/__init__.py

@@ -14,6 +14,7 @@ Sub-modules:
 import logging
 from collections import namedtuple
 
+from src.tool_security import BUILTIN_EMAIL_TOOLS
 from src.tool_utils import _truncate, get_mcp_manager, set_mcp_manager
 
 logger = logging.getLogger(__name__)
@@ -22,9 +23,15 @@ from .subprocess_tools import BashTool, PythonTool
 from .web_tools import WebSearchTool, WebFetchTool
 from .filesystem_tools import ReadFileTool, WriteFileTool, EditFileTool, LsTool, GlobTool, GrepTool, GetWorkspaceTool
 from .document_tools import CreateDocumentTool, UpdateDocumentTool, EditDocumentTool, SuggestDocumentTool, ManageDocumentTool
+from .interaction_tools import AskUserTool, UpdatePlanTool
 from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool
 from .bg_job_tools import ManageBgJobsTool
 from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool
+from .admin_tools import (
+    ADMIN_TOOL_HANDLERS,
+    do_manage_endpoints, do_manage_mcp, do_manage_webhooks,
+    do_manage_tokens, do_manage_settings,
+)
 
 TOOL_HANDLERS = {
     "bash": BashTool().execute,
@@ -43,6 +50,8 @@ TOOL_HANDLERS = {
     "suggest_document": SuggestDocumentTool().execute,
     "manage_documents": ManageDocumentTool().execute,
     "get_workspace": GetWorkspaceTool().execute,
+    "ask_user": AskUserTool().execute,
+    "update_plan": UpdatePlanTool().execute,
     "chat_with_model": ChatWithModelTool().execute,
     "ask_teacher": AskTeacherTool().execute,
     "list_models": ListModelsTool().execute,
@@ -52,6 +61,8 @@ TOOL_HANDLERS = {
     "send_to_session": SendToSessionTool().execute,
     "manage_session": ManageSessionTool().execute,
 }
+# Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings).
+TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS)
 
 # ---------------------------------------------------------------------------
 # Constants (re-exported for backward compatibility — single source of truth
@@ -76,9 +87,10 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi
              "manage_endpoints", "manage_mcp", "manage_webhooks",
              "manage_tokens", "manage_documents", "manage_settings",
              "manage_notes", "manage_calendar",
-             "resolve_contact", "manage_contact", "list_email_accounts", "send_email", "list_emails",
-             "read_email", "reply_to_email", "bulk_email", "archive_email",
-             "delete_email", "mark_email_read",
+             "resolve_contact", "manage_contact",
+             # Email tool names come from BUILTIN_EMAIL_TOOLS (unioned below)
+             # so the fence regex, dispatch, and non-admin blocklist all cover
+             # the same set.
              # Cookbook tools (LLM serving + downloads). Without these
              # entries, native function calls to e.g. list_served_models
              # are rejected as "Unknown function call" before reaching
@@ -95,7 +107,7 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi
              # Generic loopback to any UI-button endpoint (cookbook,
              # gallery, email folders, etc.) — agent uses this when
              # there's no named tool wrapper for the action.
-             "app_api"}
+             "app_api"} | BUILTIN_EMAIL_TOOLS
 
 ToolBlock = namedtuple("ToolBlock", ["tool_type", "content"])
 
@@ -138,10 +150,5 @@ from src.tool_implementations import (  # noqa: E402, F401
     do_search_chats,
     do_manage_skills,
     do_manage_tasks,
-    do_manage_endpoints,
-    do_manage_mcp,
-    do_manage_webhooks,
-    do_manage_tokens,
-    do_manage_settings,
     do_api_call,
 )

src/agent_tools/admin_tools.py

@@ -0,0 +1,792 @@
+"""Config/integration admin agent tools (TOOL_HANDLERS).
+
+Moved verbatim from tool_implementations.py as part of the tool-registry
+migration (#3629, the `admin_tools.py` bullet): manage_endpoints / manage_mcp /
+manage_webhooks / manage_tokens / manage_settings, plus manage_mcp's
+command-allowlist guard. Each impl keeps its `do_*(content, owner)` shape;
+ADMIN_TOOL_HANDLERS wraps them into registry `execute(content, ctx)` adapters
+via one factory.
+"""
+import json
+import os
+import re
+import logging
+from typing import Optional, Dict
+
+from src.tool_utils import get_mcp_manager, _parse_tool_args
+from src.tool_security import BUILTIN_EMAIL_TOOLS
+
+logger = logging.getLogger(__name__)
+
+
+async def do_manage_endpoints(content: str, owner: Optional[str] = None) -> Dict:
+    """Manage model endpoints: list, add, delete, enable, disable."""
+    from core.database import SessionLocal, ModelEndpoint
+    try:
+        args = _parse_tool_args(content)
+    except ValueError:
+        return {"error": "Invalid JSON arguments", "exit_code": 1}
+
+    action = args.get("action", "list")
+    db = SessionLocal()
+    try:
+        if action == "list":
+            eps = db.query(ModelEndpoint).all()
+            items = [{"id": e.id, "name": e.name, "base_url": e.base_url,
+                       "is_enabled": e.is_enabled} for e in eps]
+            return {"response": f"{len(items)} endpoints", "endpoints": items, "exit_code": 0}
+
+        elif action == "add":
+            import uuid as _uuid
+            name = args.get("name", "")
+            base_url = args.get("base_url", "")
+            api_key = args.get("api_key", "")
+            if not base_url:
+                return {"error": "base_url is required", "exit_code": 1}
+            eid = str(_uuid.uuid4())[:8]
+            from datetime import datetime
+            ep = ModelEndpoint(id=eid, name=name or base_url, base_url=base_url,
+                               api_key=api_key, is_enabled=True,
+                               created_at=datetime.utcnow(), updated_at=datetime.utcnow())
+            db.add(ep)
+            db.commit()
+            return {"response": f"Added endpoint '{name or base_url}' (id: {eid})", "exit_code": 0}
+
+        elif action == "delete":
+            eid = args.get("endpoint_id", "")
+            ep = db.query(ModelEndpoint).filter(ModelEndpoint.id == eid).first()
+            if not ep:
+                return {"error": f"Endpoint {eid} not found", "exit_code": 1}
+            name = ep.name
+            db.delete(ep)
+            db.commit()
+            return {"response": f"Deleted endpoint '{name}'", "exit_code": 0}
+
+        elif action in ("enable", "disable"):
+            eid = args.get("endpoint_id", "")
+            ep = db.query(ModelEndpoint).filter(ModelEndpoint.id == eid).first()
+            if not ep:
+                return {"error": f"Endpoint {eid} not found", "exit_code": 1}
+            ep.is_enabled = (action == "enable")
+            db.commit()
+            return {"response": f"Endpoint '{ep.name}' {action}d", "exit_code": 0}
+
+        else:
+            return {"error": f"Unknown action: {action}", "exit_code": 1}
+    except Exception as e:
+        logger.error(f"manage_endpoints error: {e}")
+        return {"error": str(e), "exit_code": 1}
+    finally:
+        db.close()
+
+
+# ---------------------------------------------------------------------------
+# MCP server management tool
+# ---------------------------------------------------------------------------
+
+# Parallel to routes/cookbook_helpers._validate_serve_cmd but deliberately the
+# opposite policy: that gate guards an admin-only serve command and allows
+# interpreters (python3/etc) because model-serving needs them, whereas this is
+# the model/prompt-injection-reachable manage_mcp path, so interpreters and
+# runners are denied here.
+#
+# Commands that can execute arbitrary code regardless of their arguments. These
+# are NEVER accepted on the manage_mcp agent path, even if an operator lists one
+# in ODYSSEUS_MCP_ALLOWED_COMMANDS -- a stdio server that genuinely needs an
+# interpreter or package runner must be registered via the trusted admin route.
+_MCP_DENIED_COMMANDS = frozenset({
+    "sh", "bash", "zsh", "fish", "dash", "ksh", "csh", "tcsh", "ash", "busybox",
+    "cmd", "command.com", "powershell", "pwsh",
+    "python", "pypy", "node", "nodejs", "deno", "bun", "ruby", "jruby",
+    "perl", "raku", "php", "lua", "luajit", "tclsh", "wish", "expect", "rscript",
+    "groovy", "scala", "elixir", "erl", "iex", "java", "javac", "jshell", "jbang",
+    "kotlin", "kotlinc", "dotnet", "mono", "swift", "osascript", "tsx", "ts-node",
+    "npx", "bunx", "uvx", "pipx", "npm", "pnpm", "yarn", "pip", "uv",
+    "gem", "cargo", "go", "bundle", "poetry", "conda", "mamba", "brew",
+    "apt", "apt-get", "yum", "dnf", "pacman", "apk",
+    "env", "xargs", "nohup", "setsid", "nice", "ionice", "time", "timeout",
+    "watch", "stdbuf", "unbuffer", "script", "ssh", "scp", "sshpass", "sudo",
+    "doas", "su", "make", "cmake", "docker", "podman", "kubectl", "find",
+    "awk", "gawk", "sed", "vi", "vim", "nvim", "emacs", "ed", "tee", "eval",
+})
+
+# Argv flags that make even an allowlisted binary execute inline code. Matched
+# by prefix so glued forms (-cimport os, --eval=...) are caught, not just the
+# exact-token form.
+_MCP_CODE_EXEC_SHORT_FLAGS = ("-c", "-e", "-m")
+_MCP_CODE_EXEC_LONG_FLAGS = ("--eval", "--exec", "--print", "--module", "--command", "--require")
+
+_MCP_URL_SCHEMES = ("http://", "https://", "ftp://", "ftps://", "file://", "data:", "jar:", "blob:")
+
+# Shell metacharacters refused in command/args. Args are passed as an argv list
+# (no shell), but refusing these keeps the surface narrow and obvious.
+_MCP_SHELL_METACHARS = set(";|&$`><\n\r")
+
+# Env vars that let a child process load attacker-supplied code before main().
+_MCP_DANGEROUS_ENV = frozenset({
+    "LD_PRELOAD", "LD_LIBRARY_PATH", "LD_AUDIT", "DYLD_INSERT_LIBRARIES",
+    "DYLD_LIBRARY_PATH", "DYLD_FRAMEWORK_PATH", "PYTHONPATH", "PYTHONSTARTUP",
+    "PYTHONHOME", "PYTHONEXECUTABLE", "NODE_OPTIONS", "NODE_PATH", "BASH_ENV",
+    "ENV", "SHELLOPTS", "PERL5LIB", "PERL5OPT", "RUBYOPT", "RUBYLIB", "GEM_PATH",
+    "R_PROFILE", "R_HOME", "PATH", "IFS", "PROMPT_COMMAND",
+})
+
+
+def _mcp_allowed_commands() -> set:
+    """Operator-configured allowlist of safe MCP launcher basenames for the agent
+    path. Empty by default; set ODYSSEUS_MCP_ALLOWED_COMMANDS (comma-separated)
+    to opt specific trusted binaries in. Denied commands are rejected even if
+    listed here."""
+    raw = os.environ.get("ODYSSEUS_MCP_ALLOWED_COMMANDS", "")
+    return {c.strip().lower() for c in raw.split(",") if c.strip()}
+
+
+def _validate_mcp_command(command, args, env) -> Optional[str]:
+    """Validate a model-supplied stdio MCP registration. Returns an error string
+    if it must be rejected, else None.
+
+    Closes the RCE where manage_mcp 'add' passed prompt-injection-controlled
+    command/args/env straight to a subprocess spawn (issue #438): a payload
+    smuggled into a skill description, memory entry, fetched page, or email body
+    could register a stdio server running arbitrary code as the app UID.
+    """
+    if not isinstance(command, str) or not command.strip():
+        return "command must be a non-empty string"
+    command = command.strip()
+    if "/" in command or "\\" in command:
+        return "command must be a bare executable name, not a path"
+    if any(ch in _MCP_SHELL_METACHARS for ch in command):
+        return "command contains shell metacharacters"
+    base = command.lower()
+    if base.endswith(".exe") or base.endswith(".cmd") or base.endswith(".bat"):
+        base = base.rsplit(".", 1)[0]
+    # Canonicalize a trailing version suffix so versioned aliases collapse to the
+    # family name (python3.11 -> python, node18 -> node, pip3 -> pip); both the
+    # raw basename and the canonical form are denied, so an operator cannot
+    # accidentally allowlist a runtime alias back into the path.
+    canon = re.sub(r"[-_.]?\d+(?:\.\d+)*$", "", base)
+    if base in _MCP_DENIED_COMMANDS or canon in _MCP_DENIED_COMMANDS:
+        return (
+            f"command '{command}' is not allowed on the agent MCP path: "
+            "interpreters, runtimes, package runners, and shells can execute "
+            "arbitrary code. Register such a server via the admin route instead."
+        )
+    if base not in _mcp_allowed_commands():
+        return (
+            f"command '{command}' is not in the MCP allowlist. Add it to "
+            "ODYSSEUS_MCP_ALLOWED_COMMANDS if you trust it, or register the "
+            "server via the admin route."
+        )
+
+    if args is not None:
+        if isinstance(args, str):
+            try:
+                args = json.loads(args)
+            except Exception:
+                return "args must be a JSON list"
+        if not isinstance(args, list):
+            return "args must be a list"
+        for a in args:
+            if not isinstance(a, str):
+                return "args must all be strings"
+            s = a.strip()
+            low = s.lower()
+            if any(s == f or s.startswith(f) for f in _MCP_CODE_EXEC_SHORT_FLAGS):
+                return f"arg '{a}' is a code-execution flag and is not allowed"
+            if any(low == f or low.startswith(f + "=") for f in _MCP_CODE_EXEC_LONG_FLAGS):
+                return f"arg '{a}' is a code-execution flag and is not allowed"
+            if any(low.startswith(u) for u in _MCP_URL_SCHEMES):
+                return f"arg '{a}' is a remote URL and is not allowed"
+            if any(ch in _MCP_SHELL_METACHARS for ch in a):
+                return f"arg '{a}' contains shell metacharacters"
+
+    if env:
+        if isinstance(env, str):
+            try:
+                env = json.loads(env)
+            except Exception:
+                return "env must be a JSON object"
+        if not isinstance(env, dict):
+            return "env must be an object"
+        for k in env:
+            if str(k).strip().upper() in _MCP_DANGEROUS_ENV:
+                return f"env var '{k}' can inject code into the child process and is not allowed"
+
+    return None
+
+
+async def do_manage_mcp(content: str, owner: Optional[str] = None) -> Dict:
+    """Manage MCP servers: list, add, delete, enable, disable, reconnect."""
+    try:
+        args = _parse_tool_args(content)
+    except ValueError:
+        return {"error": "Invalid JSON arguments", "exit_code": 1}
+
+    action = args.get("action", "list")
+
+    if action == "list":
+        mcp = get_mcp_manager()
+        if not mcp:
+            return {"response": "No MCP manager available", "servers": [], "exit_code": 0}
+        from core.database import SessionLocal, McpServer
+        db = SessionLocal()
+        try:
+            servers = db.query(McpServer).all()
+            items = []
+            for s in servers:
+                st = mcp.get_server_status(s.id)
+                status = st.get("status", "disconnected")
+                tool_count = st.get("tool_count", 0)
+                items.append({"id": s.id, "name": s.name, "transport": s.transport,
+                              "is_enabled": s.is_enabled, "status": status,
+                              "tool_count": tool_count})
+            return {"response": f"{len(items)} MCP servers", "servers": items, "exit_code": 0}
+        finally:
+            db.close()
+
+    elif action == "add":
+        from core.database import SessionLocal, McpServer
+        import uuid as _uuid
+        from datetime import datetime
+        name = args.get("name", "")
+        command = args.get("command", "")
+        cmd_args = args.get("args", [])
+        env = args.get("env", {})
+        if not name or not command:
+            return {"error": "name and command are required", "exit_code": 1}
+        # Validate BEFORE any DB write or spawn: a rejected registration must
+        # leave no enabled row (which would otherwise auto-reconnect on restart)
+        # and must not attempt a connection.
+        _mcp_err = _validate_mcp_command(command, cmd_args, env)
+        if _mcp_err:
+            return {"error": f"manage_mcp: refused unsafe server registration: {_mcp_err}", "exit_code": 1}
+        sid = str(_uuid.uuid4())[:8]
+        db = SessionLocal()
+        try:
+            srv = McpServer(id=sid, name=name, transport="stdio", command=command,
+                            args=json.dumps(cmd_args) if isinstance(cmd_args, list) else cmd_args,
+                            env=json.dumps(env) if isinstance(env, dict) else env,
+                            is_enabled=True, created_at=datetime.utcnow(), updated_at=datetime.utcnow())
+            db.add(srv)
+            db.commit()
+        finally:
+            db.close()
+        # Try to connect
+        mcp = get_mcp_manager()
+        tool_count = 0
+        if mcp:
+            try:
+                await mcp.connect_server(
+                    sid, name, "stdio", command=command,
+                    args=cmd_args if isinstance(cmd_args, list) else json.loads(cmd_args),
+                    env=env if isinstance(env, dict) else json.loads(env),
+                )
+                st = mcp.get_server_status(sid)
+                tool_count = st.get("tool_count", 0)
+            except Exception as e:
+                logger.warning(f"MCP connect failed for {name}: {e}")
+        return {"response": f"Added MCP server '{name}' ({tool_count} tools)", "exit_code": 0}
+
+    elif action == "delete":
+        sid = args.get("server_id", "")
+        from core.database import SessionLocal, McpServer
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == sid).first()
+            if not srv:
+                return {"error": f"Server {sid} not found", "exit_code": 1}
+            name = srv.name
+            mcp = get_mcp_manager()
+            if mcp:
+                try:
+                    await mcp.disconnect_server(sid)
+                except Exception:
+                    pass
+            db.delete(srv)
+            db.commit()
+            return {"response": f"Deleted MCP server '{name}'", "exit_code": 0}
+        finally:
+            db.close()
+
+    elif action == "reconnect":
+        sid = args.get("server_id", "")
+        mcp = get_mcp_manager()
+        if not mcp:
+            return {"error": "MCP manager not available", "exit_code": 1}
+        try:
+            await mcp.disconnect_server(sid)
+            from core.database import SessionLocal, McpServer
+            db2 = SessionLocal()
+            try:
+                srv = db2.query(McpServer).filter(McpServer.id == sid).first()
+                if srv:
+                    _args = json.loads(srv.args) if srv.args else []
+                    _env = json.loads(srv.env) if srv.env else {}
+                    await mcp.connect_server(
+                        server_id=sid,
+                        name=srv.name,
+                        transport=srv.transport,
+                        command=srv.command,
+                        args=_args,
+                        env=_env,
+                        url=srv.url,
+                    )
+                    st = mcp.get_server_status(sid)
+                    return {"response": f"Reconnected '{srv.name}' ({st.get('tool_count', 0)} tools)", "exit_code": 0}
+                return {"error": f"Server {sid} not found", "exit_code": 1}
+            finally:
+                db2.close()
+        except Exception as e:
+            return {"error": str(e), "exit_code": 1}
+
+    elif action in ("enable", "disable"):
+        sid = args.get("server_id", "")
+        from core.database import SessionLocal, McpServer
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == sid).first()
+            if not srv:
+                return {"error": f"Server {sid} not found", "exit_code": 1}
+            srv.is_enabled = (action == "enable")
+            db.commit()
+            return {"response": f"MCP server '{srv.name}' {action}d", "exit_code": 0}
+        finally:
+            db.close()
+
+    elif action == "list_tools":
+        mcp = get_mcp_manager()
+        if not mcp:
+            return {"response": "No MCP manager", "tools": [], "exit_code": 0}
+        tools = mcp.get_all_tools()
+        items = [{"name": t["name"], "server": t["server_name"],
+                  "description": t.get("description", "")[:100]} for t in tools]
+        return {"response": f"{len(items)} MCP tools available", "tools": items, "exit_code": 0}
+
+    else:
+        return {"error": f"Unknown action: {action}", "exit_code": 1}
+
+
+# ---------------------------------------------------------------------------
+# Webhook management tool
+# ---------------------------------------------------------------------------
+
+async def do_manage_webhooks(content: str, owner: Optional[str] = None) -> Dict:
+    """Manage webhooks: list, add, delete, enable, disable, test."""
+    from core.database import SessionLocal
+    try:
+        args = _parse_tool_args(content)
+    except ValueError:
+        return {"error": "Invalid JSON arguments", "exit_code": 1}
+
+    action = args.get("action", "list")
+    db = SessionLocal()
+    try:
+        from core.database import Webhook
+        if action == "list":
+            hooks = db.query(Webhook).all()
+            items = [{"id": h.id, "name": h.name, "url": h.url,
+                       "events": h.events, "is_active": h.is_active} for h in hooks]
+            return {"response": f"{len(items)} webhooks", "webhooks": items, "exit_code": 0}
+
+        elif action == "add":
+            import uuid as _uuid
+            from datetime import datetime
+            from src.webhook_manager import validate_events, validate_webhook_url
+            name = args.get("name", "")
+            url = args.get("url", "")
+            events = args.get("events", "chat.completed")
+            if not url:
+                return {"error": "url is required", "exit_code": 1}
+            try:
+                url = validate_webhook_url(url)
+                events = validate_events(events)
+            except ValueError as e:
+                return {"error": str(e), "exit_code": 1}
+            wid = str(_uuid.uuid4())[:8]
+            hook = Webhook(id=wid, name=name or url, url=url,
+                           events=events, is_active=True,
+                           created_at=datetime.utcnow(), updated_at=datetime.utcnow())
+            db.add(hook)
+            db.commit()
+            return {"response": f"Added webhook '{name or url}'", "exit_code": 0}
+
+        elif action == "delete":
+            wid = args.get("webhook_id", "")
+            hook = db.query(Webhook).filter(Webhook.id == wid).first()
+            if not hook:
+                return {"error": f"Webhook {wid} not found", "exit_code": 1}
+            name = hook.name
+            db.delete(hook)
+            db.commit()
+            return {"response": f"Deleted webhook '{name}'", "exit_code": 0}
+
+        elif action in ("enable", "disable"):
+            wid = args.get("webhook_id", "")
+            hook = db.query(Webhook).filter(Webhook.id == wid).first()
+            if not hook:
+                return {"error": f"Webhook {wid} not found", "exit_code": 1}
+            hook.is_active = (action == "enable")
+            db.commit()
+            return {"response": f"Webhook '{hook.name}' {action}d", "exit_code": 0}
+
+        else:
+            return {"error": f"Unknown action: {action}", "exit_code": 1}
+    except Exception as e:
+        logger.error(f"manage_webhooks error: {e}")
+        return {"error": str(e), "exit_code": 1}
+    finally:
+        db.close()
+
+
+# ---------------------------------------------------------------------------
+# API token management tool
+# ---------------------------------------------------------------------------
+
+async def do_manage_tokens(content: str, owner: Optional[str] = None) -> Dict:
+    """Manage API tokens: list, create, delete."""
+    from core.database import SessionLocal, ApiToken
+    try:
+        args = _parse_tool_args(content)
+    except ValueError:
+        return {"error": "Invalid JSON arguments", "exit_code": 1}
+
+    action = args.get("action", "list")
+    db = SessionLocal()
+    try:
+        if action == "list":
+            tokens = db.query(ApiToken).all()
+            items = [{"id": t.id, "name": t.name, "token_prefix": t.token_prefix + "...",
+                       "is_active": t.is_active} for t in tokens]
+            return {"response": f"{len(items)} API tokens", "tokens": items, "exit_code": 0}
+
+        elif action == "create":
+            import uuid as _uuid, secrets, bcrypt
+            from datetime import datetime
+            name = args.get("name", "API Token")
+            raw_token = secrets.token_urlsafe(32)
+            token_hash = bcrypt.hashpw(raw_token.encode(), bcrypt.gensalt()).decode()
+            tid = str(_uuid.uuid4())[:8]
+            t = ApiToken(id=tid, name=name, token_hash=token_hash,
+                         token_prefix=raw_token[:8], is_active=True,
+                         created_at=datetime.utcnow(), updated_at=datetime.utcnow())
+            db.add(t)
+            db.commit()
+            return {"response": f"Created token '{name}'", "token": raw_token, "exit_code": 0}
+
+        elif action == "delete":
+            tid = args.get("token_id", "")
+            t = db.query(ApiToken).filter(ApiToken.id == tid).first()
+            if not t:
+                return {"error": f"Token {tid} not found", "exit_code": 1}
+            name = t.name
+            db.delete(t)
+            db.commit()
+            return {"response": f"Deleted token '{name}'", "exit_code": 0}
+
+        else:
+            return {"error": f"Unknown action: {action}", "exit_code": 1}
+    except Exception as e:
+        logger.error(f"manage_tokens error: {e}")
+        return {"error": str(e), "exit_code": 1}
+    finally:
+        db.close()
+
+# ---------------------------------------------------------------------------
+# Settings/preferences management tool
+# ---------------------------------------------------------------------------
+
+async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
+    """Manage user settings and preferences."""
+    try:
+        args = _parse_tool_args(content)
+    except ValueError:
+        return {"error": "Invalid JSON arguments", "exit_code": 1}
+
+    action = args.get("action", "list")
+
+    from core.database import SessionLocal
+    db = SessionLocal()
+    try:
+        # set/get/list/delete operate on the REAL app settings (the same store
+        # the Settings panel writes), so changing a model / voice / search
+        # engine / reminder channel from chat actually takes effect.
+        from src.settings import load_settings, save_settings, DEFAULT_SETTINGS
+
+        # Secrets/credentials the agent must NOT write: kept read-only (masked)
+        # so API keys never flow through chat. User sets these in the panel.
+        _SECRET_KEYS = {
+            "brave_api_key", "google_pse_key", "google_pse_cx",
+            "tavily_api_key", "serper_api_key", "app_public_url",
+        }
+        def _is_secret(k):
+            # `token` must be a suffix, not a substring: otherwise the int
+            # setting `agent_input_token_budget` (which even has a "token budget"
+            # alias to set it from chat) is wrongly classified as a credential.
+            return (
+                k in _SECRET_KEYS
+                or k.endswith("token")
+                or any(t in k for t in ("api_key", "_key", "secret", "password"))
+            )
+
+        # Friendly aliases → real keys, so natural phrasing resolves.
+        _ALIASES_SET = {
+            "voice": "tts_voice", "tts voice": "tts_voice", "tts": "tts_enabled",
+            "text to speech": "tts_enabled", "tts provider": "tts_provider",
+            "speech speed": "tts_speed", "voice speed": "tts_speed",
+            "stt": "stt_enabled", "speech to text": "stt_enabled", "transcription": "stt_enabled",
+            "search engine": "search_provider", "search provider": "search_provider",
+            "search results": "search_result_count", "result count": "search_result_count",
+            "default model": "default_model", "chat model": "default_model",
+            "default endpoint": "default_endpoint_id",
+            "task model": "task_model", "background model": "task_model",
+            "teacher model": "teacher_model", "teacher": "teacher_enabled",
+            "utility model": "utility_model", "research model": "research_model",
+            "research max tokens": "research_max_tokens",
+            "vision model": "vision_model", "vision": "vision_enabled",
+            "image model": "image_model", "image quality": "image_quality",
+            "image gen": "image_gen_enabled", "image generation": "image_gen_enabled",
+            "reminder channel": "reminder_channel", "reminders": "reminder_channel",
+            "ntfy topic": "reminder_ntfy_topic",
+            "webhook integration": "reminder_webhook_integration_id",
+            "webhook template": "reminder_webhook_payload_template", "webhook payload": "reminder_webhook_payload_template",
+            "agent tool calls": "agent_max_tool_calls", "max tool calls": "agent_max_tool_calls",
+            "agent timeout": "agent_stream_timeout_seconds", "stream timeout": "agent_stream_timeout_seconds",
+            "token budget": "agent_input_token_budget", "input budget": "agent_input_token_budget",
+            "hard max": "agent_input_token_hard_max",
+            "token budget cap": "agent_input_token_hard_max",
+            "input budget cap": "agent_input_token_hard_max",
+        }
+        def _resolve(k):
+            k2 = (k or "").strip().lower()
+            if k2 in DEFAULT_SETTINGS:
+                return k2
+            return _ALIASES_SET.get(k2, (k or "").strip())
+
+        _ENUMS = {
+            "image_quality": ["low", "medium", "high"],
+            "reminder_channel": ["browser", "email", "ntfy", "webhook"],
+        }
+        def _coerce(value, default):
+            if isinstance(default, bool):
+                return value if isinstance(value, bool) else str(value).strip().lower() in ("true", "on", "yes", "1", "enable", "enabled")
+            if isinstance(default, int):
+                return int(value)
+            return value
+
+        def _model_slug(value: str) -> str:
+            import re as _re
+            return _re.sub(r"[^a-z0-9]+", "", (value or "").lower())
+
+        def _endpoint_model_from_cache(model_query: str):
+            """Resolve friendly model text to an enabled endpoint + real model id.
+
+            The Settings UI stores both `<prefix>_endpoint_id` and
+            `<prefix>_model`; writing only the model leaves the runtime on the
+            old endpoint. Prefer cached model lists so this stays fast/offline.
+            """
+            import json as _json
+            import re as _re
+            from core.database import ModelEndpoint
+
+            wanted = (model_query or "").strip()
+            wanted_slug = _model_slug(wanted)
+            wanted_tokens = [_model_slug(t) for t in _re.findall(r"[A-Za-z0-9]+", wanted)]
+            wanted_tokens = [t for t in wanted_tokens if t]
+            if not wanted_slug:
+                return None
+            best = None
+            for ep in db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all():
+                raw_models = []
+                try:
+                    raw_models = _json.loads(ep.cached_models or "[]") or []
+                except Exception:
+                    raw_models = []
+                # If cache is empty, still allow matching against endpoint name
+                # for callers using model@endpoint elsewhere later.
+                for mid in raw_models:
+                    mid = str(mid)
+                    mid_slug = _model_slug(mid)
+                    if not mid_slug:
+                        continue
+                    exact = mid.lower() == wanted.lower()
+                    compact_match = wanted_slug in mid_slug or mid_slug in wanted_slug
+                    token_match = bool(wanted_tokens) and all(tok in mid_slug for tok in wanted_tokens)
+                    if exact or compact_match or token_match:
+                        score = 3 if exact else (2 if compact_match else 1)
+                        if not best or score > best[0]:
+                            best = (score, ep.id, mid)
+            if best:
+                return {"endpoint_id": best[1], "model": best[2]}
+            return None
+
+        def _mask(k, v):
+            return "••••• (set in panel)" if _is_secret(k) and v else v
+
+        if action == "list":
+            s = load_settings()
+            shown = {k: _mask(k, v) for k, v in s.items() if k in DEFAULT_SETTINGS and not isinstance(v, dict)}
+            return {"response": f"{len(shown)} settings (use get/set with a key)", "settings": shown, "exit_code": 0}
+
+        elif action == "get":
+            key = _resolve(args.get("key", ""))
+            if not key:
+                return {"error": "key is required", "exit_code": 1}
+            if key not in DEFAULT_SETTINGS:
+                return {"error": f"Unknown setting '{args.get('key')}'. Use action='list' to see them.", "exit_code": 1}
+            val = load_settings().get(key, DEFAULT_SETTINGS.get(key))
+            return {"response": f"{key} = {_mask(key, val)}", "value": _mask(key, val), "exit_code": 0}
+
+        elif action == "set":
+            raw = args.get("key", "")
+            value = args.get("value")
+            if not raw:
+                return {"error": "key is required", "exit_code": 1}
+            key = _resolve(raw)
+            if key not in DEFAULT_SETTINGS:
+                return {"error": f"Unknown setting '{raw}'. Use action='list' to see available settings.", "exit_code": 1}
+            if _is_secret(key):
+                return {"response": f"'{key}' is a credential/secret. For security I can't set it from chat. Open Settings and set it there.", "exit_code": 0}
+            # Structured settings (dicts/lists like keybinds, default_model_fallbacks)
+            # have no safe scalar coercion; _coerce would pass a bare string
+            # straight through and clobber the structure. Refuse them here; they're
+            # edited in their dedicated panels. (reset/delete still restore the
+            # default structure, which is safe.)
+            if isinstance(DEFAULT_SETTINGS[key], (dict, list)):
+                return {"response": f"'{key}' is a structured setting. Edit it in its panel, not from chat. (You can reset it to default here.)", "exit_code": 0}
+            try:
+                value = _coerce(value, DEFAULT_SETTINGS[key])
+            except (ValueError, TypeError):
+                return {"error": f"'{value}' isn't a valid value for {key} (expected {type(DEFAULT_SETTINGS[key]).__name__}).", "exit_code": 1}
+            if key in _ENUMS and str(value).lower() not in _ENUMS[key]:
+                return {"error": f"{key} must be one of: {', '.join(_ENUMS[key])}.", "exit_code": 1}
+            s = load_settings()
+            s[key] = value
+            if key in {"default_model", "research_model", "utility_model", "task_model", "vision_model", "image_model"}:
+                resolved = _endpoint_model_from_cache(str(value))
+                if resolved:
+                    prefix = key[:-6]
+                    s[f"{prefix}_endpoint_id"] = resolved["endpoint_id"]
+                    s[key] = resolved["model"]
+                    value = resolved["model"]
+            save_settings(s)
+            if key.endswith("_model") and s.get(f"{key[:-6]}_endpoint_id"):
+                return {"response": f"Set {key} = {value} (endpoint {s.get(f'{key[:-6]}_endpoint_id')}).", "exit_code": 0}
+            return {"response": f"Set {key} = {value}.", "exit_code": 0}
+
+        elif action == "delete" or action == "reset":
+            key = _resolve(args.get("key", ""))
+            if key not in DEFAULT_SETTINGS:
+                return {"error": f"Unknown setting '{args.get('key')}'.", "exit_code": 1}
+            if _is_secret(key):
+                return {"response": f"'{key}' is a credential. Reset it in the panel.", "exit_code": 0}
+            s = load_settings()
+            s[key] = DEFAULT_SETTINGS[key]
+            save_settings(s)
+            return {"response": f"Reset {key} to default ({DEFAULT_SETTINGS[key]}).", "exit_code": 0}
+
+        elif action in ("disable_tool", "enable_tool", "list_tools"):
+            # Tool-toggle actions. These edit settings.json:disabled_tools
+            # (the global list read on every chat request) rather than
+            # prefs.json. Friendly aliases accepted: "shell" -> "bash",
+            # "search" -> "web_search", "browser" -> "builtin_browser",
+            # "documents" -> the document tool set, "memory" ->
+            # manage_memory, etc.
+            from src.settings import get_setting, save_settings, load_settings
+            _ALIASES = {
+                "shell": ["bash"],
+                "terminal": ["bash"],
+                "search": ["web_search", "web_fetch"],
+                "web": ["web_search", "web_fetch"],
+                "browser": ["builtin_browser"],
+                "documents": ["create_document", "edit_document", "update_document", "suggest_document"],
+                "doc": ["create_document", "edit_document", "update_document", "suggest_document"],
+                "memory": ["manage_memory"],
+                "skills": ["manage_skills"],
+                "images": ["generate_image"],
+                "image": ["generate_image"],
+                "tasks": ["manage_tasks"],
+                "notes": ["manage_notes"],
+                "calendar": ["manage_calendar"],
+                # The full built-in email tool set, in BOTH spellings: the
+                # qualified mcp__email__* names drive MCP schema hiding, the
+                # bare names drive function-schema hiding, and the runtime
+                # gate accepts either — deriving from BUILTIN_EMAIL_TOOLS
+                # keeps the toggle covering every tool the email server
+                # exposes instead of a hand-picked subset.
+                "email": sorted(BUILTIN_EMAIL_TOOLS)
+                         + [f"mcp__email__{t}" for t in sorted(BUILTIN_EMAIL_TOOLS)],
+                "research": ["web_search", "web_fetch"],  # research is a per-request flag, not a tool (closest analog)
+            }
+
+            if action == "list_tools":
+                current = get_setting("disabled_tools", []) or []
+                return {
+                    "response": (
+                        f"Currently disabled: {', '.join(current) if current else '(none)'}.\n"
+                        "Common toggles: shell (bash), search (web_search), browser, documents, "
+                        "memory, skills, images, tasks, notes, calendar, email."
+                    ),
+                    "disabled": list(current),
+                    "exit_code": 0,
+                }
+
+            tool_name = (args.get("tool") or args.get("name") or "").strip().lower()
+            if not tool_name:
+                return {"error": "tool name required (e.g. 'shell', 'search', 'bash')", "exit_code": 1}
+            targets = _ALIASES.get(tool_name, [tool_name])
+
+            settings = load_settings()
+            current = list(settings.get("disabled_tools") or [])
+            before = set(current)
+            if action == "disable_tool":
+                for t in targets:
+                    if t not in current:
+                        current.append(t)
+            else:  # enable_tool
+                current = [t for t in current if t not in targets]
+            after = set(current)
+            settings["disabled_tools"] = current
+            save_settings(settings)
+
+            verb = "Disabled" if action == "disable_tool" else "Enabled"
+            changed = sorted(after.symmetric_difference(before))
+            return {
+                "response": (
+                    f"{verb} {tool_name} ({', '.join(targets)}). "
+                    f"Now disabled: {', '.join(current) if current else '(none)'}."
+                ),
+                "changed": changed,
+                "disabled": list(current),
+                "exit_code": 0,
+            }
+
+        else:
+            return {"error": f"Unknown action: {action}", "exit_code": 1}
+    except Exception as e:
+        logger.error(f"manage_settings error: {e}")
+        return {"error": str(e), "exit_code": 1}
+    finally:
+        db.close()
+
+
+# ---------------------------------------------------------------------------
+# API call tool
+# ---------------------------------------------------------------------------
+
+
+
+# ── registry adapters ────────────────────────────────────────────────────────
+def _owner_adapter(fn):
+    """Wrap a do_*(content, owner) impl as a registry execute(content, ctx)."""
+    async def _execute(content: str, ctx: dict) -> dict:
+        return await fn(content, ctx.get("owner"))
+    return _execute
+
+
+ADMIN_TOOL_HANDLERS = {
+    "manage_endpoints": _owner_adapter(do_manage_endpoints),
+    "manage_mcp": _owner_adapter(do_manage_mcp),
+    "manage_webhooks": _owner_adapter(do_manage_webhooks),
+    "manage_tokens": _owner_adapter(do_manage_tokens),
+    "manage_settings": _owner_adapter(do_manage_settings),
+}

src/agent_tools/document_tools.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict, List, Optional
 import logging
 import re
-import json
 from src.constants import MAX_READ_CHARS
+from src.tool_utils import _parse_tool_args
 
 logger = logging.getLogger(__name__)
 
@@ -154,38 +154,6 @@ def _coerce_email_document_content(existing: str, incoming: str) -> str:
         body = new
     return header.rstrip() + "\n---\n" + body
 
-def _parse_tool_args(content):
-    """Parse a tool-call argument blob.
-
-    Accepts either a JSON string or an already-decoded dict. Unwraps the
-    common `{"body": {...}}` envelope that smaller models emit when they
-    read tool descriptions like "Body is JSON: {...}" literally — they
-    pass `body` as a field name rather than treating it as a noun.
-
-    Returns a dict on success, raises ValueError on bad JSON.
-    """
-    if isinstance(content, str):
-        try:
-            args = json.loads(content) if content.strip() else {}
-        except (json.JSONDecodeError, TypeError) as e:
-            raise ValueError(str(e))
-    elif isinstance(content, dict):
-        args = content
-    else:
-        args = {}
-    # Unwrap {"body": {...}} envelope — but only if `body` is the sole key
-    # and points at a dict. We don't want to clobber a legitimate `body`
-    # field on tools where it's a real arg (e.g. send_email body text).
-    if (
-        isinstance(args, dict)
-        and len(args) == 1
-        and "body" in args
-        and isinstance(args["body"], dict)
-        and "action" in args["body"]  # extra safety: only unwrap if the inner dict looks like a tool call
-    ):
-        args = args["body"]
-    return args
-
 def parse_edit_blocks(content: str) -> list:
     """Parse <<<FIND>>>...<<<REPLACE>>>...<<<END>>> blocks."""
     edits = []
@@ -217,6 +185,71 @@ def parse_suggest_blocks(content: str) -> list:
     return suggestions
 
 
+def _pdf_source_upload_id(content: str) -> Optional[str]:
+    try:
+        from src.pdf_form_doc import find_source_upload_id
+        return find_source_upload_id(content or "")
+    except Exception:
+        return None
+
+
+def _strip_pdf_editor_markers(content: str) -> str:
+    """Turn a PDF-wrapper markdown doc into ordinary editable markdown.
+
+    PDF docs use hidden HTML comments for source-upload links, form fields, and
+    page annotations. Those comments are necessary for rendering/exporting the
+    original PDF, but they make a derived AI text edit keep showing the original
+    PDF preview. Remove only the editor plumbing and keep the readable text.
+    """
+    text = content or ""
+    text = re.sub(r'(?im)^\s*<!--\s*pdf(?:_form)?_source\s+[^>]*-->\s*\n*', '', text)
+    text = re.sub(r'\s*<!--\s*field=[^>]*-->', '', text)
+    text = re.sub(r'\s*<!--\s*annotation\s+[^>]*-->', '', text)
+    return text.strip()
+
+
+def _create_pdf_text_derivative(db, *, source_doc, content: str, owner: Optional[str], summary: str) -> dict:
+    import uuid
+    from src.database import Document, DocumentVersion
+
+    clean = _strip_pdf_editor_markers(content)
+    title_base = (getattr(source_doc, "title", None) or "PDF").strip()
+    title = title_base if title_base.lower().endswith("edited") else f"{title_base} edited"
+    doc_id = str(uuid.uuid4())
+    ver_id = str(uuid.uuid4())
+    new_doc = Document(
+        id=doc_id,
+        session_id=getattr(source_doc, "session_id", None),
+        title=title,
+        language="markdown",
+        current_content=clean,
+        version_count=1,
+        is_active=True,
+        owner=owner if owner is not None else getattr(source_doc, "owner", None),
+    )
+    ver = DocumentVersion(
+        id=ver_id,
+        document_id=doc_id,
+        version_number=1,
+        content=clean,
+        summary=summary,
+        source="ai",
+    )
+    db.add(new_doc)
+    db.add(ver)
+    db.commit()
+    set_active_document(doc_id)
+    return {
+        "action": "create",
+        "doc_id": doc_id,
+        "title": title,
+        "language": "markdown",
+        "content": clean,
+        "version": 1,
+        "source_doc_id": getattr(source_doc, "id", None),
+    }
+
+
 class CreateDocumentTool:
     async def execute(self, content: str, ctx: dict) -> dict:
         """Create a new document. Supports two formats:
@@ -364,6 +397,15 @@ class UpdateDocumentTool:
             if is_email_doc:
                 doc.language = "email"
 
+            if not is_email_doc and _pdf_source_upload_id(doc.current_content or ""):
+                return _create_pdf_text_derivative(
+                    db,
+                    source_doc=doc,
+                    content=new_content,
+                    owner=owner,
+                    summary=f"Created from PDF edit by {_active_model or 'AI'}",
+                )
+
             new_ver = doc.version_count + 1
             ver = DocumentVersion(
                 id=str(uuid.uuid4()),
@@ -448,6 +490,15 @@ class EditDocumentTool:
             if applied == 0:
                 return {"error": f"No edits applied — none of the FIND blocks matched the document content (skipped {skipped})"}
 
+            if _pdf_source_upload_id(doc.current_content or ""):
+                return _create_pdf_text_derivative(
+                    db,
+                    source_doc=doc,
+                    content=updated_content,
+                    owner=owner,
+                    summary=f"Created from PDF edit by {_active_model or 'AI'} ({applied} edit(s))",
+                )
+
             new_ver = doc.version_count + 1
             ver = DocumentVersion(
                 id=str(uuid.uuid4()),
@@ -596,9 +647,20 @@ class ManageDocumentTool:
                 if not doc:
                     return {"error": f"Document '{doc_id}' not found", "exit_code": 1}
                 body = doc.current_content or ""
-                preview_limit = int(args.get("limit", MAX_READ_CHARS))
-                truncated = len(body) > preview_limit
-                preview = body[:preview_limit] + (f"\n... (truncated, {len(body)} chars total)" if truncated else "")
+                try:
+                    preview_limit = max(1, min(int(args.get("limit", MAX_READ_CHARS)), MAX_READ_CHARS))
+                except (TypeError, ValueError):
+                    preview_limit = MAX_READ_CHARS
+                try:
+                    offset = max(0, int(args.get("offset", 0) or 0))
+                except (TypeError, ValueError):
+                    offset = 0
+                offset = min(offset, len(body))
+                end = min(offset + preview_limit, len(body))
+                truncated = end < len(body)
+                preview = body[offset:end]
+                if truncated:
+                    preview += f"\n... (truncated, {len(body)} chars total; next_offset={end})"
                 anchor = f"[{doc.title}](#document-{doc.id})"
                 return {
                     "response": f"{anchor} — click to open in editor.\n\n```{doc.language or ''}\n{preview}\n```",
@@ -609,6 +671,8 @@ class ManageDocumentTool:
                         "size": len(body),
                         "content": preview,
                         "truncated": truncated,
+                        "offset": offset,
+                        "next_offset": end if truncated else None,
                     },
                     "exit_code": 0,
                 }
@@ -641,4 +705,4 @@ class ManageDocumentTool:
             logger.error(f"manage_documents error: {e}")
             return {"error": str(e), "exit_code": 1}
         finally:
-            db.close()
+            db.close()

src/agent_tools/filesystem_tools.py

@@ -186,6 +186,21 @@ class WriteFileTool:
         lines = content.split("\n", 1)
         raw_path = lines[0].strip()
         body = lines[1] if len(lines) > 1 else ""
+        # Decode JSON-object args (the fenced inline-args shape
+        # ```write_file {"path": "...", "content": "..."}```), matching
+        # ReadFileTool above. Without this the whole JSON string becomes the
+        # path and the file is written under a garbage name. This is the live
+        # path: there is no filesystem MCP server, so write_file always runs
+        # here via _direct_fallback, not through _build_mcp_args.
+        _stripped = content.strip()
+        if _stripped.startswith("{"):
+            try:
+                _a = json.loads(_stripped)
+                if isinstance(_a, dict) and "path" in _a:
+                    raw_path = str(_a.get("path", "")).strip()
+                    body = str(_a.get("content", ""))
+            except (json.JSONDecodeError, TypeError, ValueError):
+                pass
         try:
             path = _resolve_tool_path(raw_path)
         except ValueError as e:
@@ -266,7 +281,13 @@ class LsTool:
 
 class GlobTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
+        from src.tool_execution import (
+            _SENSITIVE_BASENAMES,
+            _is_sensitive_path,
+            _resolve_tool_path,
+            _resolve_search_root,
+            _truncate,
+        )
         args = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -288,11 +309,30 @@ class GlobTool:
             base = os.path.abspath(root)
             if not os.path.isdir(base):
                 return None, f"glob: {root}: not a directory"
+            rbase = os.path.realpath(base)
             norm_pat = pattern.replace("\\", "/")
             # Fast path: literal pattern (no wildcards) → direct path lookup.
             if not any(c in norm_pat for c in "*?["):
-                cand = os.path.normpath(os.path.join(base, norm_pat))
-                if os.path.exists(cand):
+                cand = os.path.realpath(os.path.join(base, norm_pat))
+                # Keep the literal lookup inside the search root. os.path.join
+                # lets an absolute pattern (or one containing ../) escape `base`,
+                # which would turn glob into an existence/path oracle for
+                # arbitrary host files — bypassing the workspace/allowlist
+                # confinement that _resolve_search_root applies to the root.
+                # An escaping literal falls through to the walk, which only ever
+                # yields paths under base.
+                nbase = os.path.normcase(rbase)
+                try:
+                    inside = cand == rbase or os.path.commonpath(
+                        [os.path.normcase(cand), nbase]
+                    ) == nbase
+                except ValueError:
+                    inside = False
+                # A literal that names a deny-listed sensitive file (.env,
+                # .ssh/id_rsa, …) falls through to the walk, which skips it —
+                # otherwise glob would surface secret paths that read_file /
+                # grep already refuse to touch.
+                if inside and os.path.exists(cand) and not _is_sensitive_path(cand):
                     return [cand], None
                 # Literal not at exact path — fall through to walk so
                 # e.g. "foo.py" still matches at any depth (like rglob).
@@ -304,11 +344,20 @@ class GlobTool:
                 for dp, dns, fns in os.walk(base):
                     # Prune skipped dirs before descending (unlike rglob which
                     # descends first then filters — fatal on large node_modules).
-                    dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
+                    # Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
+                    # never enumerates the keys/tokens inside them.
+                    dns[:] = [
+                        d for d in dns
+                        if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
+                    ]
                     for name in fns + dns:
                         full = os.path.join(dp, name)
                         rel = os.path.relpath(full, base).replace(os.sep, "/")
                         if regex.fullmatch(rel) or regex.fullmatch(name):
+                            # Skip deny-listed sensitive files (.env, id_rsa,
+                            # known_hosts, …) the same way grep does.
+                            if _is_sensitive_path(os.path.realpath(full)):
+                                continue
                             try:
                                 mtime = os.stat(full).st_mtime
                             except OSError:
@@ -333,7 +382,13 @@ class GlobTool:
 
 class GrepTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
+        from src.tool_execution import (
+            _SENSITIVE_FILE_PATTERNS,
+            _is_sensitive_path,
+            _resolve_tool_path,
+            _resolve_search_root,
+            _truncate,
+        )
         args: Dict[str, Any] = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -369,6 +424,8 @@ class GrepTool:
                     cmd.append("--ignore-case")
                 if glob_pat:
                     cmd += ["--glob", glob_pat]
+                for _pat in _SENSITIVE_FILE_PATTERNS:
+                    cmd += ["--glob", f"!*{_pat}*"]
                 for _d in _CODENAV_SKIP_DIRS:
                     cmd += ["--glob", f"!**/{_d}/**"]
                 cmd += ["--regexp", pattern, root]
@@ -399,6 +456,8 @@ class GrepTool:
             for fp in file_iter:
                 if len(hits) >= max_hits:
                     break
+                if _is_sensitive_path(os.path.realpath(fp)):
+                    continue
                 try:
                     with open(fp, "r", encoding="utf-8", errors="strict") as f:
                         for i, line in enumerate(f, 1):

src/agent_tools/interaction_tools.py

@@ -0,0 +1,95 @@
+import json
+import logging
+
+logger = logging.getLogger(__name__)
+
+class AskUserTool:
+    async def execute(self, content, ctx):
+        """
+        ask_user: the agent poses a multiple-choice question to the user to get a
+        decision/clarification. This is a pure UI-control marker — no subprocess,
+        no filesystem. It returns an `ask_user` payload that the agent loop turns
+        into an `ask_user` SSE event and then ENDS the turn, so the chat waits for
+        the user's selection (their choice arrives as the next message).
+        """
+        question, options, multi = "", [], False
+        raw = (content or "").strip()
+        try:
+            parsed = json.loads(raw) if raw else {}
+        except (ValueError, TypeError):
+            parsed = {}
+
+        if isinstance(parsed, dict):
+            question = str(parsed.get("question", "")).strip()
+            multi = bool(parsed.get("multi") or parsed.get("multiSelect"))
+            for opt in (parsed.get("options") or []):
+                if isinstance(opt, dict):
+                    label = str(opt.get("label", "")).strip()
+                    descr = str(opt.get("description", "")).strip()
+                elif isinstance(opt, str):
+                    label, descr = opt.strip(), ""
+                else:
+                    continue
+                if label:
+                    options.append({"label": label, "description": descr})
+        else:
+            question = raw
+
+        if not question or len(options) < 2:
+            return "ask_user: invalid", {
+                "error": (
+                    "ask_user needs a non-empty `question` and at least 2 `options` "
+                    "(each an object with a `label`, optional `description`)."
+                ),
+                "exit_code": 1,
+            }
+
+        options = options[:6]  # keep the choice list sane
+        desc = f"ask_user: {question[:80]}"
+        labels = ", ".join(o["label"] for o in options)
+        result = {
+            "ask_user": {"question": question, "options": options, "multi": multi},
+            "output": f"Asked the user: {question}\nOptions: {labels}\nAwaiting their selection.",
+            "exit_code": 0,
+        }
+        logger.info("Tool executed: %s (%d options, multi=%s)", desc, len(options), multi)
+        return desc, result
+
+class UpdatePlanTool:
+    async def execute(self, content, ctx):
+        """
+        update_plan: the agent writes back to the active plan — tick an item done
+        or revise steps (e.g. when the user asks to change something). Pure UI
+        marker: returns a `plan_update` payload the agent loop turns into a
+        `plan_update` SSE event; the frontend replaces the stored plan and refreshes
+        the docked plan window. Does NOT end the turn.
+        """
+        raw = (content or "").strip()
+        plan = ""
+        try:
+            parsed = json.loads(raw) if raw else {}
+        except (ValueError, TypeError):
+            parsed = {}
+
+        if isinstance(parsed, dict) and parsed.get("plan"):
+            plan = str(parsed.get("plan", "")).strip()
+        else:
+            plan = raw
+
+        if not plan:
+            return "update_plan: invalid", {
+                "error": "update_plan needs a non-empty `plan` (the full updated checklist as markdown).",
+                "exit_code": 1,
+            }
+
+        plan = plan[:8192]
+        done = plan.count("- [x]") + plan.count("- [X]")
+        total = done + plan.count("- [ ]")
+        desc = f"update_plan: {done}/{total} done" if total else "update_plan"
+        result = {
+            "plan_update": {"plan": plan},
+            "output": f"Plan updated ({done}/{total} steps complete)." if total else "Plan updated.",
+            "exit_code": 0,
+        }
+        logger.info("Tool executed: %s", desc)
+        return desc, result

src/agent_tools/model_interaction_tools.py

@@ -10,6 +10,7 @@ Shared helpers that still live in ``src.ai_interaction`` and are used by tools
 not yet migrated (``_resolve_model``, ``AI_CHAT_TIMEOUT``) are imported lazily
 inside the functions to avoid an import cycle at module load.
 """
+import asyncio
 import logging
 from typing import Dict, Optional
 
@@ -46,7 +47,7 @@ async def chat_with_model(content: str, session_id: Optional[str] = None, owner:
         return {"error": "No message provided (line 2+ is the message)"}
 
     try:
-        url, model, headers = _resolve_model(model_spec, owner=owner)
+        url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=owner)
     except ValueError as e:
         return {"error": str(e)}
 
@@ -90,7 +91,7 @@ async def ask_teacher(content: str, session_id: Optional[str] = None, owner: Opt
             return {"error": "No teacher model configured. Specify a model name or set teacher_model in settings."}
 
     try:
-        url, model, headers = _resolve_model(model_spec, owner=owner)
+        url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=owner)
     except ValueError as e:
         return {"error": str(e)}
 

src/agent_tools/session_tools.py

@@ -8,6 +8,7 @@ The session manager is a runtime-set singleton in src.ai_interaction, so each
 function fetches it via get_session_manager() (imported here); _resolve_model and
 AI_CHAT_TIMEOUT are reused from there too.
 """
+import asyncio
 import json
 import logging
 import uuid
@@ -40,7 +41,7 @@ async def create_session(content: str, session_id: Optional[str] = None, owner:
         return {"error": "Session name cannot be empty"}
 
     try:
-        url, model, headers = _resolve_model(model_spec, owner=owner)
+        url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=owner)
     except ValueError as e:
         return {"error": str(e)}
 
@@ -103,6 +104,8 @@ async def list_sessions(content: str, session_id: Optional[str] = None, owner: O
         sessions = _session_manager.get_sessions_for_user(owner)
         rows = []
         for sid, sess in sessions.items():
+            if (sess.name or "").startswith("SFT trace batch"):
+                continue
             if keyword and keyword not in (sess.name or "").lower():
                 continue
             db_row = db_rows.get(sid)
@@ -191,6 +194,25 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner:
     try:
         # Build context from session history
         context = sess.get_context_messages()
+        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
+        model = str(getattr(sess, "model", "") or "")
+        if model == "fixture-tool-model" or "host.docker.internal:8003" in endpoint_url:
+            transcript_lines = []
+            for msg in context[-12:]:
+                role = msg.get("role", "unknown")
+                text = (msg.get("content") or "").strip()
+                if text:
+                    transcript_lines.append(f"{role}: {text}")
+            transcript = "\n".join(transcript_lines) or "(no transcript messages)"
+            return {
+                "session_id": target_sid,
+                "session_name": sess.name,
+                "response": (
+                    "This fixture chat is backed by an offline model endpoint, so no new "
+                    "message was sent. Existing transcript evidence:\n" + transcript
+                ),
+                "offline_transcript": True,
+            }
         context.append({"role": "user", "content": message})
 
         response = await llm_call_async(

src/agent_tools/web_tools.py

@@ -7,6 +7,7 @@ from src.constants import MAX_OUTPUT_CHARS
 class WebSearchTool:
     async def execute(self, content: str, ctx: dict) -> dict:
         from src.search import comprehensive_web_search
+        progress_cb = ctx.get("progress_cb") if isinstance(ctx, dict) else None
         raw = content.strip()
         query = raw
         time_filter = None
@@ -37,18 +38,39 @@ class WebSearchTool:
             elif " news" in q_lc or q_lc.startswith("news ") or q_lc.endswith(" news"):
                 time_filter = "week"
         loop = asyncio.get_running_loop()
-        text, sources = await asyncio.wait_for(
-            loop.run_in_executor(
-                None,
-                lambda: comprehensive_web_search(
-                    query,
-                    max_pages=max_pages,
-                    time_filter=time_filter,
-                    return_sources=True,
+        if progress_cb:
+            await progress_cb({
+                "elapsed_s": 0,
+                "tail": f"Searching web for: {query[:160]}",
+            })
+        try:
+            text, sources = await asyncio.wait_for(
+                loop.run_in_executor(
+                    None,
+                    lambda: comprehensive_web_search(
+                        query,
+                        max_pages=max_pages,
+                        time_filter=time_filter,
+                        return_sources=True,
+                    ),
                 ),
-            ),
-            timeout=30,
-        )
+                timeout=30,
+            )
+        except asyncio.TimeoutError:
+            return {
+                "error": f"web_search timed out after 30s: {query[:200]}",
+                "exit_code": 1,
+            }
+        except Exception as e:
+            return {
+                "error": f"web_search failed: {type(e).__name__}: {str(e) or 'no details'}",
+                "exit_code": 1,
+            }
+        if progress_cb:
+            await progress_cb({
+                "elapsed_s": 30,
+                "tail": "Search completed; preparing sources.",
+            })
         output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text
         if sources:
             output += "\n\n<!-- SOURCES:" + json.dumps(sources) + " -->"

src/ai_interaction.py

@@ -14,6 +14,7 @@ These are agent tools — the LLM writes fenced code blocks and they execute
 through the standard agent_tools.py pipeline.
 """
 
+import asyncio
 import json
 import logging
 import uuid
@@ -134,7 +135,8 @@ def _resolve_model(spec: str, owner: Optional[str] = None) -> Tuple[str, str, Di
                         r = httpx.get(models_url, headers=headers, timeout=5)
                         r.raise_for_status()
                         data = r.json()
-                        model_ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
+                        items = data if isinstance(data, list) else (data.get("data") or [])
+                        model_ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
                         if not model_ids:
                             model_ids = [
                                 m.get("name") or m.get("model")
@@ -228,7 +230,7 @@ async def do_pipeline(content: str, session_id: Optional[str] = None, owner: Opt
         if not model_spec or not instruction:
             return {"error": f"Step {i + 1}: both 'model' and 'instruction' are required"}
         try:
-            url, model, headers = _resolve_model(model_spec, owner=owner)
+            url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=owner)
             resolved.append((url, model, headers, instruction))
         except ValueError as e:
             return {"error": f"Step {i + 1}: {e}"}
@@ -431,13 +433,23 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
             return {"error": "Search needs line 2: query"}
         query = lines[1].strip()
         memories = _memory_manager.load(owner=owner)
+        query_lower = query.lower()
+        exact_results = [m for m in memories if query_lower in (m.get("text", "").lower())]
 
         if hasattr(_memory_manager, 'get_relevant_memories'):
-            results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+            vector_results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
         else:
-            # Fallback: simple text search
-            query_lower = query.lower()
-            results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
+            vector_results = []
+        seen = set()
+        results = []
+        for m in [*exact_results, *vector_results]:
+            mid = m.get("id")
+            if mid in seen:
+                continue
+            seen.add(mid)
+            results.append(m)
+            if len(results) >= 20:
+                break
 
         if not results:
             return {"results": f"No memories found matching '{query}'."}
@@ -453,8 +465,6 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
         return {"error": f"Unknown action '{action}'. Use: list, add, edit, delete, search"}
 
 
-
-
 # ---------------------------------------------------------------------------
 # RAG management tool
 # ---------------------------------------------------------------------------
@@ -625,7 +635,7 @@ async def do_ui_control(content: str, session_id: Optional[str] = None, owner: O
 
         # Resolve the model to validate it exists
         try:
-            url, model_id, headers = _resolve_model(model_spec, owner=owner)
+            url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=owner)
         except ValueError as e:
             return {"error": str(e)}
 
@@ -915,7 +925,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
     if not model_spec:
         for candidate in ("gpt-image-1.5", "gpt-image-1", "dall-e-3"):
             try:
-                _resolve_model(candidate, owner=owner)
+                await asyncio.to_thread(_resolve_model, candidate, owner=owner)
                 model_spec = candidate
                 break
             except ValueError:
@@ -942,7 +952,9 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
                         try:
                             _r = _req.get(_ibase + "/models", timeout=3)
                             _r.raise_for_status()
-                            _mids = [m.get("id") for m in (_r.json().get("data") or []) if m.get("id")]
+                            _data = _r.json()
+                            _ditems = _data if isinstance(_data, list) else (_data.get("data") or [])
+                            _mids = [m.get("id") for m in _ditems if isinstance(m, dict) and m.get("id")]
                             if _mids:
                                 model_spec = _mids[0]
                                 break
@@ -957,7 +969,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne
 
     # Resolve the model to find the right endpoint
     try:
-        url, model_id, headers = _resolve_model(model_spec, owner=owner)
+        url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=owner)
     except ValueError:
         return {"error": f"No endpoint found with image model '{model_spec}'. "
                 "Configure an OpenAI-compatible endpoint with image generation support."}

src/api_key_manager.py

@@ -81,11 +81,26 @@ class APIKeyManager:
         keys stay encrypted. Loading via load() first would decrypt them and
         write them back as plaintext, which then fails to decrypt on the next
         load() and silently drops those providers.
+
+        Uses atomic write (temp file + os.replace) so a crash, disk-full, or
+        mid-write error never truncates the existing keys file.
         """
         keys = self._load_raw()
         keys[provider] = self.encrypt_api_key(api_key)
-        with open(self.api_keys_file, 'w', encoding="utf-8") as f:
-            json.dump(keys, f)
+        tmp_file = self.api_keys_file + ".tmp"
+        try:
+            with open(tmp_file, 'w', encoding="utf-8") as f:
+                json.dump(keys, f)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(tmp_file, self.api_keys_file)
+        except OSError:
+            # Clean up temp file on failure; re-raise so callers see the error
+            try:
+                os.remove(tmp_file)
+            except OSError:
+                pass
+            raise
 
     def load(self) -> Dict[str, str]:
         """Load and decrypt API keys"""

src/app_helpers.py

@@ -1,6 +1,13 @@
 # src/app_helpers.py
-import os
 import base64
+import logging
+import os
+
+from fastapi import HTTPException
+from fastapi.responses import HTMLResponse
+from starlette.requests import Request
+
+logger = logging.getLogger(__name__)
 
 def read_if_exists(path: str) -> str:
     """Read file if it exists, return empty string otherwise."""
@@ -20,6 +27,28 @@ def abs_join(base_dir: str, rel: str) -> str:
     """Join paths and return absolute path."""
     return os.path.abspath(os.path.join(base_dir, rel))
 
+def serve_html_with_nonce(request: Request, file_path: str) -> HTMLResponse:
+    """Read an app-bundled HTML page and inject the CSP nonce into inline <script> tags.
+
+    Callers pass fixed, server-owned template paths (index/login/backgrounds),
+    never a client-supplied path. So any read failure here — a missing file
+    (broken deployment) or a permission/IO error — is a server fault, not a
+    client "not found": map all of them to a logged 500 so a missing core
+    template surfaces in 5xx alerting instead of hiding behind a 404. If a
+    future caller serves a client-influenced path where 404 is correct, branch
+    that at the call site rather than defaulting this shared helper to 404.
+    """
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            html = f.read()
+    except OSError:
+        logger.exception("Failed to read page %s", file_path)
+        raise HTTPException(500, "Internal server error")
+    nonce = getattr(request.state, "csp_nonce", "")
+    html = html.replace("{{CSP_NONCE}}", nonce)
+    return HTMLResponse(html)
+
+
 def inside_base_dir(base_dir: str, path: str) -> bool:
     """Check if path is inside base directory."""
     if not isinstance(base_dir, str) or not isinstance(path, str):

src/app_initializer.py

@@ -68,8 +68,10 @@ def initialize_managers(base_dir: str, rag_manager=None) -> Dict[str, Any]:
                     logger.info(f"Rebuilt memory vector index from {len(existing)} existing entries")
             logger.info("MemoryVectorStore initialized")
         else:
+            # Keep the unhealthy object (do NOT reset to None): consumers gate on
+            # `.healthy`, and service_health.chromadb_health() needs a present
+            # object to report DEGRADED/DOWN instead of DISABLED ("not configured").
             logger.warning("MemoryVectorStore DEGRADED: ChromaDB vector memory unavailable")
-            memory_vector = None
     except Exception as e:
         logger.warning(f"MemoryVectorStore DEGRADED: {e}")
         memory_vector = None

src/builtin_mcp.py

@@ -89,6 +89,21 @@ _BUILTIN_NPX_SERVERS = {
 MCP_DISABLED = os.environ.get("ODYSSEUS_DISABLE_MCP", "").lower() in ("1", "true", "yes")
 
 
+# Strong references to the fire-and-forget startup tasks scheduled below.
+# asyncio only keeps weak references to tasks created via create_task, so
+# without this the GC can collect a task mid-execution and the server
+# registration silently never runs. Mirrors _spawn_bg in routes/chat_helpers.py.
+_BG_TASKS: set[asyncio.Task] = set()
+
+
+def _spawn_bg(coro) -> asyncio.Task:
+    """Schedule a background task and hold a strong reference until it finishes."""
+    task = asyncio.create_task(coro)
+    _BG_TASKS.add(task)
+    task.add_done_callback(_BG_TASKS.discard)
+    return task
+
+
 async def register_builtin_servers(mcp_manager):
     """Connect all built-in MCP servers to the manager."""
     if MCP_DISABLED:
@@ -123,7 +138,7 @@ async def register_builtin_servers(mcp_manager):
         if not os.path.exists(script_path):
             logger.warning(f"Built-in MCP server script not found: {script_path}")
             continue
-        asyncio.create_task(_connect_python_server(server_id, script_path, name))
+        _spawn_bg(_connect_python_server(server_id, script_path, name))
 
     # Register NPX-based servers in the background (they take longer to start)
     npx_path = _find_npx()
@@ -175,7 +190,7 @@ async def register_builtin_servers(mcp_manager):
             except BaseException as e:
                 logger.warning(f"Built-in NPX server {cfg['name']} error: {type(e).__name__}: {e}")
 
-    asyncio.create_task(_start_npx_servers())
+    _spawn_bg(_start_npx_servers())
 
 
 def _npx_package_from_args(args):
@@ -233,6 +248,15 @@ async def _is_npx_package_cached(npx_path, package_spec, timeout_s=5):
         except Exception:
             pass
         return False
+    except asyncio.CancelledError:
+        # The probe was cancelled (e.g. app shutdown). Reap the child so it
+        # isn't orphaned, then propagate the cancellation.
+        try:
+            proc.kill()
+            await proc.wait()
+        except Exception:
+            pass
+        raise
     return proc.returncode == 0 and bool(stdout.strip())
 
 

src/caldav_sync.py

@@ -25,6 +25,7 @@ Design notes:
 import asyncio
 import hashlib
 import ipaddress
+import json
 import logging
 import os
 import socket
@@ -274,6 +275,7 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i
     # the integrations form still works, sync just no-ops with an error.
     from caldav.lib.error import AuthorizationError, NotFoundError
     from core.database import CalendarCal, CalendarEvent, SessionLocal
+    from routes.calendar_routes import _ensure_positive_duration
 
     result = {"calendars": 0, "events": 0, "deleted": 0, "errors": []}
 
@@ -390,6 +392,11 @@ def _sync_blocking(owner: str, url: str, username: str, password: str, account_i
                             end_dt = start_dt + timedelta(days=1)
                         else:
                             end_dt = start_dt + timedelta(hours=1)
+                        # A synced event with DTEND <= DTSTART (e.g. a single-day
+                        # all-day event whose source wrote DTEND equal to DTSTART)
+                        # would be stored zero-duration and silently dropped by the
+                        # list_events overlap filter. Clamp to a positive span.
+                        end_dt = _ensure_positive_duration(start_dt, end_dt, all_day)
 
                         # is_utc reflects whether the source carried a TZ
                         # we converted from. All-day = no TZ semantics.
@@ -494,6 +501,7 @@ def _event_payload(ev) -> dict:
         "all_day": ev.all_day,
         "is_utc": ev.is_utc,
         "rrule": ev.rrule or "",
+        "recurrence_exdates": json.loads(ev.recurrence_exdates or "[]") if getattr(ev, "recurrence_exdates", "") else [],
     }
 
 

src/caldav_writeback.py

@@ -33,7 +33,8 @@ def build_event_ical(ev: dict) -> str:
     """Serialize a local event dict to a VCALENDAR/VEVENT iCalendar string.
 
     ``ev`` keys: uid, summary, description, location, dtstart (datetime),
-    dtend (datetime), all_day (bool), is_utc (bool), rrule (str).
+    dtend (datetime), all_day (bool), is_utc (bool), rrule (str),
+    recurrence_exdates (list[str]).
     Mirrors how the pull path interprets is_utc/all_day so a round-trip is stable.
     """
     from icalendar import Calendar, Event as iEvent
@@ -70,6 +71,15 @@ def build_event_ical(ev: dict) -> str:
             ve.add("rrule", vRecur.from_ical(ev["rrule"]))
         except Exception:
             logger.debug("CalDAV write-back: skipping unparseable rrule %r", ev.get("rrule"))
+    for exdate in ev.get("recurrence_exdates") or []:
+        try:
+            if ev.get("all_day"):
+                ve.add("exdate", datetime.strptime(exdate[:10], "%Y-%m-%d").date())
+            else:
+                dt = datetime.strptime(exdate[:16], "%Y-%m-%dT%H:%M")
+                ve.add("exdate", dt.replace(tzinfo=timezone.utc) if ev.get("is_utc") else dt)
+        except Exception:
+            logger.debug("CalDAV write-back: skipping unparseable exdate %r", exdate)
 
     cal.add_component(ve)
     return cal.to_ical().decode("utf-8")

src/chat_handler.py

@@ -29,6 +29,34 @@ from src.youtube_handler import (
 logger = logging.getLogger(__name__)
 
 
+def _sync_upload_vision_to_gallery(file_info: Dict[str, Any], owner: Optional[str], text: str) -> None:
+    file_hash = (file_info or {}).get("hash")
+    if not file_hash or not text:
+        return
+    try:
+        from core.database import GalleryImage, SessionLocal
+        db = SessionLocal()
+        try:
+            q = db.query(GalleryImage).filter(
+                GalleryImage.file_hash == file_hash,
+                GalleryImage.is_active == True,  # noqa: E712
+            )
+            if owner:
+                q = q.filter(GalleryImage.owner == owner)
+            img = q.first()
+            if not img:
+                return
+            img.caption = text.strip()
+            db.commit()
+        except Exception:
+            db.rollback()
+            raise
+        finally:
+            db.close()
+    except Exception as e:
+        logger.warning("Failed to sync upload vision text to gallery: %s", e)
+
+
 class ChatHandler:
     """Handles chat operations for both streaming and non-streaming endpoints."""
 
@@ -207,6 +235,7 @@ class ChatHandler:
                                     _vtext = _vf.read().strip()
                                 if _vtext:
                                     enhanced_message += f"\n[User-corrected caption / OCR for this image — treat as authoritative]:\n{_vtext}"
+                                    _sync_upload_vision_to_gallery(file_info, owner, _vtext)
                                     _m = meta_by_id.get(att_id)
                                     if _m is not None:
                                         _m["vision"] = _vtext
@@ -226,6 +255,7 @@ class ChatHandler:
                                     cached_desc = _vf.read().strip()
                                 if cached_desc and not cached_desc.startswith("["):
                                     vl_desc = cached_desc
+                                    _sync_upload_vision_to_gallery(file_info, owner, vl_desc)
                             except Exception:
                                 vl_desc = None
                         if not vl_desc:
@@ -237,6 +267,7 @@ class ChatHandler:
                                     os.makedirs(os.path.join(UPLOAD_DIR, ".vision"), exist_ok=True)
                                     with open(_vcache, "w", encoding="utf-8") as _vf:
                                         _vf.write(vl_desc)
+                                    _sync_upload_vision_to_gallery(file_info, owner, vl_desc)
                                 except Exception:
                                     pass
                         enhanced_message = f"{enhanced_message}\n\n[Image: {file_info['name']}]\n{vl_desc}"

src/chat_processor.py

@@ -12,6 +12,45 @@ from src.prompt_security import UNTRUSTED_CONTEXT_POLICY, untrusted_context_mess
 
 logger = logging.getLogger(__name__)
 
+
+def _clean_search_query(query: str, max_len: int = 200) -> str:
+    """Strip fenced code blocks from a search query while preserving inline
+    code text.
+
+    This is a focused, defensive cleanup for the *final* web-search query
+    selected in ``build_context_preface`` (issue #4547): regardless of whether
+    the query came from the LLM-generated path (#4557) or the first-line
+    fallback, residual fenced / inline markdown should not leak into the search
+    call. Rather than using regex (which is brittle and strips inline code
+    text like ``git reset`` from the query), we render the query to HTML via
+    ``markdown`` and parse it with ``BeautifulSoup`` so that:
+
+    * ``<pre>`` blocks (fenced / indented code) are removed entirely.
+    * ``<code>`` elements (inline code) are preserved as plain text.
+
+    Both libraries are already project dependencies. The result is whitespace
+    collapsed and truncated to ``max_len``; an all-code input collapses to an
+    empty string, which the caller treats as "no query".
+    """
+    import markdown as _md
+    from bs4 import BeautifulSoup as _BS
+
+    html = _md.markdown(query, extensions=["fenced_code"])
+    soup = _BS(html, "html.parser")
+
+    # Remove fenced / indented code blocks.
+    for pre in soup.find_all("pre"):
+        pre.decompose()
+
+    # Preserve inline code by unwrapping <code> to text.
+    for code in soup.find_all("code"):
+        code.replace_with(code.get_text())
+
+    text = soup.get_text(" ", strip=True)
+    text = re.sub(r"\s+", " ", text)
+    return text[:max_len]
+
+
 # ── Stopwords & tokenizer ──
 
 _STOPWORDS = frozenset(
@@ -280,10 +319,61 @@ class ChatProcessor:
         web_sources = []
         if use_web:
             try:
-                web_context, web_sources = comprehensive_web_search(
-                    message, time_filter=time_filter, return_sources=True
-                )
-                preface.append(untrusted_context_message("web search results", web_context))
+                from src.llm_core import llm_call
+
+                t_url, t_model, t_headers = session.endpoint_url, session.model, session.headers
+
+                # Default fallback is the first non-empty line of the original user message
+                fallback_query = next((line.strip() for line in message.split("\n") if line.strip()), "")
+                search_query = fallback_query
+
+                try:
+                    generated_query = llm_call(
+                        t_url,
+                        t_model,
+                        [
+                            {
+                                "role": "system",
+                                "content": (
+                                    "Extract a concise search query from the user's message. "
+                                    "Reply ONLY with the query."
+                                ),
+                            },
+                            {"role": "user", "content": message},
+                        ],
+                        headers=t_headers,
+                        temperature=0.1,
+                        max_tokens=50,
+                        timeout=15,
+                    ).strip()
+
+                    if generated_query:
+                        # LLM successfully generated a non-empty query -> use the generated query
+                        search_query = generated_query
+                    else:
+                        # LLM returned an empty or whitespace-only query -> fall back to original query
+                        logger.warning("LLM generated an empty search query, using fallback.")
+                except Exception as e:
+                    # LLM failed (exception/error) -> fall back to original user query
+                    logger.warning(f"Failed to generate search query via LLM, using fallback: {e}")
+
+                search_query = " ".join(search_query.split())
+                if len(search_query) > 150:
+                    search_query = search_query[:150].strip()
+
+                # Defensive cleanup of the final selected query (interim fix
+                # for #4547): strip any residual fenced/inline markdown so that
+                # neither the generated query nor the first-line fallback leaks
+                # fences or backticks into the search call. No-op on clean
+                # generated queries; collapses to "" when the query is all code.
+                search_query = _clean_search_query(search_query, max_len=150)
+
+                if search_query:
+                    # Execute web search using the final selected query
+                    web_context, web_sources = comprehensive_web_search(
+                        search_query, time_filter=time_filter, return_sources=True
+                    )
+                    preface.append(untrusted_context_message("web search results", web_context))
             except Exception as e:
                 logger.error(f"Web search failed: {e}")
                 preface.append({"role": "system", "content": "Web search encountered an error and could not retrieve results."})

src/constants.py

@@ -4,7 +4,7 @@ import os
 
 from src.runtime_paths import get_app_root, get_default_data_dir
 
-APP_VERSION = "1.0.0"
+APP_VERSION = "1.0.1"
 
 # Base paths
 BASE_DIR = os.path.join(get_app_root(), "")

src/context_budget.py

@@ -18,6 +18,13 @@ DEFAULT_BUDGET = 6000
 DEFAULT_HEADROOM = 0.85
 
 
+def _int_or_zero(value) -> int:
+    try:
+        return int(value or 0)
+    except (TypeError, ValueError):
+        return 0
+
+
 def compute_input_token_budget(
     configured: int,
     context_length: int,
@@ -48,8 +55,8 @@ def compute_input_token_budget(
         - When the window is unknown (context_length <= 0), use the conservative
           ``default`` budget and do NOT scale off the fallback.
     """
-    configured = int(configured or 0)
-    context_length = int(context_length or 0)
+    configured = _int_or_zero(configured)
+    context_length = _int_or_zero(context_length)
 
     if explicit and configured > 0:
         return min(configured, context_length) if context_length > 0 else configured

src/cookbook_serve_lifecycle.py

@@ -37,6 +37,13 @@ async def _delete_endpoint_for_task(task: dict) -> None:
     the picker (probe goes offline; chats still try to route there) and
     the user has to delete it by hand in Settings -> Endpoints.
     """
+    endpoint_id = (task.get("_endpointId") or task.get("endpointId") or "").strip()
+    if not endpoint_id:
+        logger.info(
+            "cookbook_serve_lifecycle: task %s has no endpoint id; skipping endpoint deletion",
+            task.get("sessionId") or task.get("id") or "",
+        )
+        return
     import re as _re
     payload = task.get("payload") or {}
     cmd = str(payload.get("_cmd") or "")
@@ -66,13 +73,10 @@ async def _delete_endpoint_for_task(task: dict) -> None:
             if r.status_code >= 400:
                 return
             eps = r.json() if r.content else []
-            # Prefer exact URL match; fall back to host:port substring so we
-            # still catch the case where 0.0.0.0 vs the registered host
-            # representation diverged.
-            ep = next((e for e in eps if e.get("base_url") == base_url), None)
-            if not ep:
-                hostport = f"{host}:{port}"
-                ep = next((e for e in eps if hostport in (e.get("base_url") or "")), None)
+            # Delete only the endpoint created by this scheduled serve. URL
+            # matching is unsafe because a later scheduled serve can reuse the
+            # same host:port after an older task has gone stale.
+            ep = next((e for e in eps if e.get("id") == endpoint_id), None)
             if ep:
                 await client.delete(
                     f"{internal_api_base()}/api/model-endpoints/{ep['id']}",

src/document_actions.py

@@ -6,7 +6,7 @@ Reusable document actions callable from both REST routes and the task scheduler.
 
 import logging
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 
 logger = logging.getLogger(__name__)
 
@@ -77,10 +77,28 @@ async def run_document_tidy(owner: str) -> str:
         deleted = 0
         kept = 0
         survivors = []  # docs that pass the junk rules, considered for dedup
+        now = datetime.now(timezone.utc)
 
         for doc in docs:
+            created = doc.created_at
+            if created and created.tzinfo is None:
+                created = created.replace(tzinfo=timezone.utc)
+
+            # Skip freshly created documents to avoid deleting them while the user is actively editing
+            if created and (now - created).total_seconds() < 900:  # 15 minutes
+                survivors.append(doc)
+                continue
+
             content = (doc.current_content or "").strip()
             title = (doc.title or "").strip().lower()
+            is_fresh_empty = (
+                not content
+                and created is not None
+                and (now - created).total_seconds() < 1800
+            )
+            if is_fresh_empty:
+                survivors.append(doc)
+                continue
 
             # Strip markdown noise to get "real" character count
             stripped = re.sub(r"^#{1,6}\s+", "", content, flags=re.MULTILINE)  # headers

src/embeddings.py

@@ -55,6 +55,8 @@ class EmbeddingClient:
         # of stalling startup ~30s per probe. Read stays generous for a real
         # endpoint (embedding a short string returns in well under a second).
         self._client = httpx.Client(timeout=httpx.Timeout(connect=3.0, read=10.0, write=5.0, pool=3.0))
+        self._batch_size = max(1, int(os.getenv("EMBEDDING_BATCH_SIZE", "8")))
+        self._max_chars = max(200, int(os.getenv("EMBEDDING_MAX_CHARS", "900")))
 
     def get_sentence_embedding_dimension(self) -> int:
         """Probe the endpoint for embedding dimension if not yet known."""
@@ -73,23 +75,10 @@ class EmbeddingClient:
         if not texts:
             return np.array([], dtype="float32")
 
-        # Batch in chunks of 64 to avoid oversized requests
         all_vecs = []
-        for i in range(0, len(texts), 64):
-            batch = texts[i : i + 64]
-            resp = self._client.post(
-                self.url,
-                headers={"Authorization": f"Bearer {self.api_key}"} if self.api_key else {},
-                json={"input": batch, "model": self.model},
-            )
-            resp.raise_for_status()
-            data = resp.json()
-
-            # OpenAI format: {"data": [{"embedding": [...], "index": 0}, ...]}
-            embeddings = data.get("data", [])
-            embeddings.sort(key=lambda e: e.get("index", 0))
-            for emb in embeddings:
-                all_vecs.append(emb["embedding"])
+        for i in range(0, len(texts), self._batch_size):
+            batch = texts[i : i + self._batch_size]
+            all_vecs.extend(self._embed_batch(batch))
 
         vecs = np.array(all_vecs, dtype="float32")
 
@@ -103,6 +92,42 @@ class EmbeddingClient:
 
         return vecs
 
+    def _embed_batch(self, batch: List[str]) -> List[List[float]]:
+        try:
+            return self._post_embeddings(batch)
+        except httpx.HTTPStatusError as e:
+            status = e.response.status_code if e.response is not None else None
+            if status != 400:
+                raise
+            if len(batch) > 1:
+                vecs = []
+                for text in batch:
+                    vecs.extend(self._embed_batch([text]))
+                return vecs
+            text = batch[0]
+            trimmed = text[: self._max_chars]
+            if trimmed != text:
+                logger.warning(
+                    "Embedding input exceeded endpoint context; retrying with %d chars",
+                    len(trimmed),
+                )
+                return self._post_embeddings([trimmed])
+            raise
+
+    def _post_embeddings(self, batch: List[str]) -> List[List[float]]:
+        resp = self._client.post(
+            self.url,
+            headers={"Authorization": f"Bearer {self.api_key}"} if self.api_key else {},
+            json={"input": batch, "model": self.model},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+        # OpenAI format: {"data": [{"embedding": [...], "index": 0}, ...]}
+        embeddings = data.get("data", [])
+        embeddings.sort(key=lambda e: e.get("index", 0))
+        return [emb["embedding"] for emb in embeddings]
+
 
 class FastEmbedClient:
     """Local embedding client using fastembed (ONNX). No external service needed."""

src/endpoint_resolver.py

@@ -424,6 +424,9 @@ def resolve_utility_fallback_candidates(owner: Optional[str] = None) -> list:
         settings = load_settings()
         utility_ep = (get_user_setting("utility_endpoint_id", owner or "", settings.get("utility_endpoint_id", "")) or "").strip()
         if not utility_ep:
+            utility_chain = get_user_setting("utility_model_fallbacks", owner or "", settings.get("utility_model_fallbacks") or []) or []
+            if utility_chain:
+                return _resolve_fallback_candidates("utility_model_fallbacks", owner=owner)
             return _resolve_fallback_candidates("default_model_fallbacks", owner=owner)
     except Exception:
         pass

src/exceptions.py

@@ -1,29 +1,22 @@
 # src/exceptions.py
-"""Custom exceptions for the application."""
+"""Backward-compatible shim — the single source of truth is core/exceptions.py.
 
-class SessionNotFoundError(Exception):
-    """Raised when a requested session is not found."""
-    def __init__(self, session_id: str):
-        self.session_id = session_id
-        super().__init__(f"Session '{session_id}' not found")
+Historically this module was a byte-for-byte duplicate of core/exceptions.py,
+which is the canonical definition (imported by app.py, core/__init__.py, and
+routes/chat_routes.py). To kill the drift, this now simply re-exports the
+exception classes from core.exceptions so there is exactly one place that
+defines them. Existing `from src.exceptions import ...` callers keep working.
+"""
+from core.exceptions import (  # noqa: F401
+    SessionNotFoundError,
+    InvalidFileUploadError,
+    LLMServiceError,
+    WebSearchError,
+)
 
-class InvalidFileUploadError(Exception):
-    """Raised when a file upload fails validation."""
-    def __init__(self, message: str, filename: str = None):
-        self.filename = filename
-        self.message = message
-        super().__init__(message)
-
-class LLMServiceError(Exception):
-    """Raised when there is an error communicating with the LLM service."""
-    def __init__(self, message: str, endpoint: str = None):
-        self.endpoint = endpoint
-        self.message = message
-        super().__init__(message)
-
-class WebSearchError(Exception):
-    """Raised when there is an error with web search functionality."""
-    def __init__(self, message: str, query: str = None):
-        self.query = query
-        self.message = message
-        super().__init__(message)
+__all__ = [
+    "SessionNotFoundError",
+    "InvalidFileUploadError",
+    "LLMServiceError",
+    "WebSearchError",
+]

src/host_docker_access.py

@@ -0,0 +1,62 @@
+"""Policy checks for explicit host Docker access from a container."""
+
+import os
+import stat
+from collections.abc import Mapping
+
+
+HOST_DOCKER_ENV_VAR = "ODYSSEUS_ENABLE_HOST_DOCKER"
+HOST_DOCKER_SOCKET_PATH = "/var/run/docker.sock"
+
+HOST_DOCKER_ACCESS_HINT = (
+    "Local Docker daemon access is disabled inside the Odysseus container; a "
+    "Docker CLI alone is not enough. Default Docker Compose intentionally does "
+    "not mount the host Docker socket. Raw socket access is high-trust and can "
+    "grant broad control over the host Docker daemon. If you accept that risk, "
+    "enable docker/host-docker.yml. Remote server Docker workflows over SSH "
+    "remain preferred."
+)
+
+
+def running_in_container(
+    dockerenv_path: str = "/.dockerenv",
+    cgroup_path: str = "/proc/1/cgroup",
+) -> bool:
+    if os.path.exists(dockerenv_path):
+        return True
+    try:
+        with open(cgroup_path, "r", encoding="utf-8") as handle:
+            contents = handle.read()
+    except OSError:
+        return False
+    return any(token in contents for token in ("docker", "containerd", "kubepods"))
+
+
+def host_docker_access_enabled(
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+    *,
+    environ: Mapping[str, str] | None = None,
+) -> bool:
+    env = os.environ if environ is None else environ
+    if env.get(HOST_DOCKER_ENV_VAR, "").strip().lower() != "true":
+        return False
+    try:
+        mode = os.stat(socket_path).st_mode
+    except OSError:
+        return False
+    return stat.S_ISSOCK(mode)
+
+
+def local_docker_available(
+    *,
+    cli_available: bool,
+    in_container: bool | None = None,
+    environ: Mapping[str, str] | None = None,
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+) -> bool:
+    if not cli_available:
+        return False
+    containerized = running_in_container() if in_container is None else in_container
+    if not containerized:
+        return True
+    return host_docker_access_enabled(socket_path, environ=environ)

src/interactive_gate.py

@@ -0,0 +1,193 @@
+"""Foreground activity gate for background work.
+
+Background tasks are allowed to run only after normal UI/API traffic has
+settled. This keeps scheduled jobs and email pollers from competing with the
+user opening Odysseus, Cookbook, email, documents, notes, or other panels.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager
+import os
+import time
+
+
+_ACTIVE_REQUESTS = 0
+_LAST_ACTIVITY = 0.0
+_LAST_BROWSER_ACTIVITY = 0.0
+_COND: asyncio.Condition | None = None
+
+
+def _enabled() -> bool:
+    return os.getenv("BACKGROUND_TASK_FOREGROUND_GATE", "true").lower() not in {"0", "false", "no", "off"}
+
+
+def _quiet_seconds() -> float:
+    try:
+        return max(0.0, float(os.getenv("BACKGROUND_TASK_QUIET_MS", "1500")) / 1000.0)
+    except Exception:
+        return 1.5
+
+
+def _max_wait_seconds() -> float:
+    """0 means wait indefinitely until the UI is quiet."""
+    try:
+        return max(0.0, float(os.getenv("BACKGROUND_TASK_MAX_WAIT_SECONDS", "0")))
+    except Exception:
+        return 0.0
+
+
+def _browser_active_seconds() -> float:
+    """How long a visible Odysseus browser heartbeat blocks background tasks."""
+    try:
+        return max(0.0, float(os.getenv("BACKGROUND_TASK_BROWSER_ACTIVE_SECONDS", "45")))
+    except Exception:
+        return 45.0
+
+
+def _condition() -> asyncio.Condition:
+    global _COND
+    if _COND is None:
+        _COND = asyncio.Condition()
+    return _COND
+
+
+_PASSIVE_EXACT_PATHS = {
+    "/api/activity/heartbeat",
+    "/api/tasks/notifications",
+    "/api/research/active",
+    "/api/email/urgency-state",
+}
+
+_PASSIVE_PREFIXES = (
+    "/api/chat/stream_status",
+    "/api/health",
+    "/api/prefs",
+)
+
+
+def should_track_interactive_request(path: str, method: str = "GET") -> bool:
+    if not _enabled():
+        return False
+    if (method or "").upper() == "OPTIONS":
+        return False
+    if path in _PASSIVE_EXACT_PATHS:
+        return False
+    if any(path.startswith(prefix) for prefix in _PASSIVE_PREFIXES):
+        return False
+    return True
+
+
+async def mark_browser_activity() -> None:
+    """Record that an authenticated browser tab is visibly using Odysseus."""
+    global _LAST_BROWSER_ACTIVITY
+    if not _enabled():
+        return
+    cond = _condition()
+    async with cond:
+        _LAST_BROWSER_ACTIVITY = time.monotonic()
+        cond.notify_all()
+
+
+def _has_recent_browser_activity(now: float | None = None) -> bool:
+    ttl = _browser_active_seconds()
+    if ttl <= 0 or _LAST_BROWSER_ACTIVITY <= 0:
+        return False
+    return ((now if now is not None else time.monotonic()) - _LAST_BROWSER_ACTIVITY) < ttl
+
+
+def has_foreground_activity(now: float | None = None) -> bool:
+    """Return True when foreground browser/model work should stop background jobs.
+
+    This is intentionally narrower than `wait_for_interactive_quiet`: active
+    request tracking is good for delaying task startup, but a running task
+    should not cancel itself just because the UI polls a passive endpoint.
+    Browser heartbeats and active chat streams are the durable "user is here"
+    signals.
+    """
+    if not _enabled():
+        return False
+    t = now if now is not None else time.monotonic()
+    return _has_recent_browser_activity(t) or _has_active_chat_stream()
+
+
+def _has_active_chat_stream() -> bool:
+    """Best-effort check for foreground model work that outlives HTTP requests.
+
+    Chat/agent streams are detached from the browser SSE so a stream can keep
+    running after the request that started it has returned. Background LLM
+    tasks must still wait for those runs; otherwise helpers like email
+    auto-translate compete with the user's active chat on the same local model.
+    """
+    try:
+        from routes import chat_routes as _chat_routes
+        active_streams = getattr(_chat_routes, "_active_streams", {}) or {}
+        if active_streams:
+            return True
+    except Exception:
+        pass
+    try:
+        from src import agent_runs
+        runs = getattr(agent_runs, "_RUNS", {}) or {}
+        return any(getattr(run, "status", None) == "running" for run in runs.values())
+    except Exception:
+        return False
+
+
+@asynccontextmanager
+async def track_interactive_request(path: str = "", method: str = ""):
+    global _ACTIVE_REQUESTS, _LAST_ACTIVITY
+    if not _enabled():
+        yield
+        return
+
+    cond = _condition()
+    async with cond:
+        _ACTIVE_REQUESTS += 1
+        _LAST_ACTIVITY = time.monotonic()
+        cond.notify_all()
+    try:
+        yield
+    finally:
+        async with cond:
+            _ACTIVE_REQUESTS = max(0, _ACTIVE_REQUESTS - 1)
+            _LAST_ACTIVITY = time.monotonic()
+            cond.notify_all()
+
+
+async def wait_for_interactive_quiet(label: str = "") -> bool:
+    """Wait until foreground requests have stopped for the configured window.
+
+    Returns True if the caller had to wait at all. The label is intentionally
+    only for future logging/debugging so callers can keep their code simple.
+    """
+    if not _enabled():
+        return False
+
+    quiet = _quiet_seconds()
+    max_wait = _max_wait_seconds()
+    deadline = time.monotonic() + max_wait if max_wait > 0 else None
+    cond = _condition()
+    waited = False
+
+    while True:
+        async with cond:
+            now = time.monotonic()
+            quiet_remaining = quiet - (now - _LAST_ACTIVITY)
+            active_stream = _has_active_chat_stream()
+            browser_active = _has_recent_browser_activity(now)
+            if _ACTIVE_REQUESTS <= 0 and quiet_remaining <= 0 and not active_stream and not browser_active:
+                return waited
+
+            waited = True
+            timeout = 0.25 if (_ACTIVE_REQUESTS > 0 or active_stream or browser_active) else min(max(quiet_remaining, 0.05), 0.5)
+            if deadline is not None:
+                remaining = deadline - now
+                if remaining <= 0:
+                    return waited
+                timeout = min(timeout, remaining)
+            try:
+                await asyncio.wait_for(cond.wait(), timeout=timeout)
+            except asyncio.TimeoutError:
+                pass

src/llm_core.py

@@ -110,6 +110,18 @@ _HARMONY_MARKERS = (
 )
 _HARMONY_MAX_MARKER_LEN = max(len(marker) for marker in _HARMONY_MARKERS)
 
+_VISIBLE_CHAT_TEMPLATE_ARTIFACT_RE = re.compile(
+    r"(?:\|end\|)+\|?assistan(?:t)?\|?"
+    r"|\|assistan(?:t)?\|"
+    r"|<\|im_start\|>\s*assistant"
+    r"|<\|im_end\|>",
+    re.IGNORECASE,
+)
+
+
+def _strip_visible_chat_template_artifacts(text: str) -> str:
+    return _VISIBLE_CHAT_TEMPLATE_ARTIFACT_RE.sub("", text or "")
+
 
 def _harmony_suffix_hold_len(text: str) -> int:
     """Return how many trailing chars could be the start of a harmony marker."""
@@ -345,43 +357,114 @@ def _normalize_ollama_url(url: str) -> str:
     return base.rstrip("/") + "/chat"
 
 
-def _ollama_normalize_tool_messages(messages: List[Dict]) -> List[Dict]:
+def _normalize_openai_chat_url(url: str) -> str:
+    """Ensure an OpenAI-compatible base URL points at /chat/completions."""
+    base = (url or "").strip().rstrip("/")
+    if not base:
+        return base
+    if base.endswith("/chat/completions") or base.endswith("/completions"):
+        return base
+    if base.endswith("/models"):
+        base = base[: -len("/models")].rstrip("/")
+    return base + "/chat/completions"
+
+
+def _ollama_normalize_messages(messages: List[Dict]) -> List[Dict]:
     """Adapt Odysseus' canonical OpenAI-style messages to native Ollama /api/chat.
 
-    Odysseus carries assistant tool calls in the OpenAI shape, where
-    `function.arguments` is a JSON *string*. Native Ollama expects it to be a
-    JSON *object*; given the string it fails the whole request with HTTP 400
-    "Value looks like object, but can't find closing '}' symbol", which aborts
-    every follow-up (tool-result) round. Parse the arguments back into an object
-    here, on a shallow copy, leaving non-tool messages untouched. The opaque
-    Gemini `extra_content` (thought_signature) is dropped — it is meaningless to
-    Ollama and only matters when the conversation is replayed to Gemini.
+    Two shape mismatches silently break requests:
+
+    1. Tool calls: Odysseus carries `function.arguments` as a JSON *string*.
+       Native Ollama expects a JSON *object* and rejects the string form with
+       HTTP 400 ("Value looks like object, but can't find closing '}' symbol"),
+       aborting every follow-up (tool-result) round. Parse the arguments back
+       into an object here, on a shallow copy, leaving non-tool messages
+       untouched. The opaque Gemini `extra_content` (thought_signature) is
+       dropped — it is meaningless to Ollama and only matters when the
+       conversation is replayed to Gemini.
+
+    2. Images (issue #4723): Odysseus carries multimodal user content as an
+       OpenAI-style list ``[{type: "text", ...}, {type: "image_url",
+       image_url: {url: "data:image/...;base64,XXX"}}, ...]``. Native Ollama
+       does not accept a list for ``content`` — it wants ``content`` as a
+       string plus a separate ``images`` array of raw base64 strings (no
+       ``data:`` prefix). Without this conversion the image blocks pass
+       through untouched, the vision-capable model never sees the picture,
+       and the user gets "I can't see any image" even though the request
+       succeeded.
     """
     out: List[Dict] = []
     for m in messages or []:
-        tcs = m.get("tool_calls") if isinstance(m, dict) else None
-        if not tcs:
+        if not isinstance(m, dict):
             out.append(m)
             continue
-        new_calls = []
-        for tc in tcs:
-            fn = tc.get("function") or {}
-            args = fn.get("arguments")
-            if isinstance(args, str):
-                try:
-                    args = json.loads(args) if args.strip() else {}
-                except (json.JSONDecodeError, TypeError):
-                    args = {}
-            call: Dict = {"function": {"name": fn.get("name", ""), "arguments": args or {}}}
-            if tc.get("id"):
-                call["id"] = tc["id"]
-            new_calls.append(call)
+
         nm = dict(m)
-        nm["tool_calls"] = new_calls
+
+        # 1. Tool-call argument strings -> objects.
+        tcs = nm.get("tool_calls")
+        if tcs:
+            new_calls = []
+            for tc in tcs:
+                fn = tc.get("function") or {}
+                args = fn.get("arguments")
+                if isinstance(args, str):
+                    try:
+                        args = json.loads(args) if args.strip() else {}
+                    except (json.JSONDecodeError, TypeError):
+                        args = {}
+                call: Dict = {"function": {"name": fn.get("name", ""), "arguments": args or {}}}
+                if tc.get("id"):
+                    call["id"] = tc["id"]
+                new_calls.append(call)
+            nm["tool_calls"] = new_calls
+
+        # 2. Multimodal content list -> native content string + images array.
+        content = nm.get("content")
+        if isinstance(content, list):
+            text_parts: List[str] = []
+            images: List[str] = list(nm.get("images") or [])
+            for block in content:
+                if not isinstance(block, dict):
+                    continue
+                btype = block.get("type")
+                if btype == "text":
+                    t = block.get("text")
+                    if t:
+                        text_parts.append(str(t))
+                elif btype == "image_url":
+                    url = (block.get("image_url") or {}).get("url", "")
+                    if not url:
+                        continue
+                    if url.startswith("data:"):
+                        # Strip the ``data:[...];base64,`` prefix — native
+                        # Ollama wants only the base64 bytes.
+                        _, _, b64 = url.partition(",")
+                        if b64:
+                            images.append(b64)
+                    else:
+                        # Native Ollama images[] is base64-only; it does
+                        # not fetch HTTP URLs.  Skip unsupported schemes
+                        # rather than sending a non-base64 string that the
+                        # model silently ignores.
+                        logger.warning(
+                            "Skipping non-data image_url (Ollama images[] "
+                            "requires base64): %s",
+                            url[:80],
+                        )
+            nm["content"] = "\n".join(text_parts).strip()
+            if images:
+                nm["images"] = images
+
         out.append(nm)
     return out
 
 
+# Backward-compatible alias for callers/tests that imported the older name
+# (it only handled tool messages originally — issue #4723 broadened scope).
+_ollama_normalize_tool_messages = _ollama_normalize_messages
+
+
 def _build_ollama_payload(
     model: str,
     messages: List[Dict],
@@ -404,7 +487,7 @@ def _build_ollama_payload(
     """
     payload: Dict = {
         "model": model,
-        "messages": _ollama_normalize_tool_messages(messages),
+        "messages": _ollama_normalize_messages(messages),
         "stream": stream,
     }
     options: Dict = {}
@@ -618,6 +701,10 @@ def _detect_provider(url: str) -> str:
     from src.copilot import is_copilot_base
     if is_copilot_base(url):
         return "copilot"
+    if _host_match(url, "cerebras.ai"):
+        return "cerebras"
+    if _host_match(url, "mistral.ai"):
+        return "mistral"
     return "openai"
 
 
@@ -702,6 +789,8 @@ def _provider_label(url: str) -> str:
     if is_chatgpt_subscription_base(url): return "ChatGPT Subscription"
     from src.copilot import is_copilot_base
     if is_copilot_base(url): return "GitHub Copilot"
+    if _host_match(url, "cerebras.ai"):
+        return "cerebras"
     if _host_match(url, "mistral.ai"): return "Mistral"
     if _host_match(url, "deepseek.com"): return "DeepSeek"
     if _host_match(url, "nvidia.com"): return "NVIDIA"
@@ -716,10 +805,17 @@ def _provider_label(url: str) -> str:
             pass
     if _is_ollama_native_url(url): return "Ollama"
     try:
-        host = (urlparse(url).hostname or "").lower()
+        _parsed_local = urlparse(url)
+        host = (_parsed_local.hostname or "").lower()
+        port = _parsed_local.port
     except Exception:
         return "provider"
     if host in {"localhost", "127.0.0.1", "::1", "0.0.0.0"}:
+        # A port alone is not authoritative: vLLM, SGLang, llama.cpp and plain
+        # OpenAI-compatible servers all routinely share 8000/8080, so naming the
+        # serving tool from the port here would mislabel real setups. The tool is
+        # identified by probing llama-server's native /props endpoint during
+        # discovery (see ModelDiscovery._fingerprint_provider); this stays neutral.
         return "local endpoint"
     return host or "provider"
 
@@ -906,8 +1002,18 @@ def _anthropic_rejects_temperature(model: str) -> bool:
         return False
     return (int(match.group(1)), int(match.group(2))) >= (4, 7)
 
+# Reasoning effort level sent to Mistral thinking-capable models. Mistral's
+# API accepts "high", "medium", "low", "none" — see
+# https://docs.mistral.ai/capabilities/reasoning/. Override via env var
+# ODYSSEUS_MISTRAL_REASONING_EFFORT (e.g. set to "medium" for cheaper chat).
+_MISTRAL_REASONING_EFFORT = os.getenv("ODYSSEUS_MISTRAL_REASONING_EFFORT", "high")
+
 # Models that support structured thinking — may output </think> without opening tag
-_THINKING_MODEL_PATTERNS = ("qwen3", "qwq", "deepseek-r1", "deepseek-reasoner", "minimax", "m2-reap", "gemma")
+_THINKING_MODEL_PATTERNS = (
+    "qwen3", "qwq", "deepseek-r1", "deepseek-reasoner", "minimax",
+    "m2-reap", "gemma", "stepfun", "step-3", "step3",
+    "magistral", "mistral-small", "mistral-medium",
+)
 
 def _supports_thinking(model: str) -> bool:
     """Check if model supports structured thinking output."""
@@ -916,6 +1022,38 @@ def _supports_thinking(model: str) -> bool:
     m = model.lower()
     return any(p in m for p in _THINKING_MODEL_PATTERNS)
 
+def _normalize_mistral_content(content):
+    """Mistral returns content as a structured array when reasoning is on:
+        [{"type": "thinking", "thinking": [{"type": "text", "text": "..."}], "closed": true},
+         {"type": "text", "text": "...final answer..."}]
+    Convert to (text, thinking) tuple of plain strings. Pass through strings
+    unchanged so non-Mistral OpenAI-compat endpoints are unaffected.
+    """
+    if isinstance(content, str):
+        return content, ""
+    if not isinstance(content, list):
+        return "", ""
+    text_parts = []
+    thinking_parts = []
+    for block in content:
+        if not isinstance(block, dict):
+            continue
+        btype = block.get("type")
+        if btype == "text":
+            t = block.get("text", "")
+            if t:
+                text_parts.append(t)
+        elif btype == "thinking":
+            inner = block.get("thinking", [])
+            if isinstance(inner, list):
+                for tb in inner:
+                    if isinstance(tb, dict) and tb.get("text"):
+                        thinking_parts.append(tb["text"])
+            elif isinstance(inner, str):
+                thinking_parts.append(inner)
+    return "".join(text_parts), "".join(thinking_parts)
+
+
 def _convert_openai_content_to_anthropic(content):
     """Convert OpenAI multimodal content blocks to Anthropic format.
 
@@ -1086,6 +1224,25 @@ def _as_content_blocks(content) -> List[Dict]:
     return []
 
 
+def _is_untrusted_context_content(content) -> bool:
+    if isinstance(content, str):
+        return (
+            content.startswith("UNTRUSTED SOURCE DATA\n")
+            or "<<<UNTRUSTED_SOURCE_DATA>>>" in content
+        )
+    if isinstance(content, list):
+        return any(
+            isinstance(block, dict)
+            and block.get("type") == "text"
+            and _is_untrusted_context_content(block.get("text") or "")
+            for block in content
+        )
+    return False
+
+
+_REFERENCE_CONTEXT_BOUNDARY = "Reference context received."
+
+
 def _sanitize_llm_messages(messages: List[Dict]) -> List[Dict]:
     """Strip Odysseus-only metadata before sending messages to providers.
 
@@ -1198,6 +1355,10 @@ def _sanitize_llm_messages(messages: List[Dict]) -> List[Dict]:
 
         last = merged[-1]
         if last.get("role") == "user" and item.get("role") == "user":
+            if _is_untrusted_context_content(last.get("content")):
+                merged.append({"role": "assistant", "content": _REFERENCE_CONTEXT_BOUNDARY})
+                merged.append(item)
+                continue
             last_copy = dict(last)
             lc = last_copy.get("content")
             ic = item.get("content")
@@ -1224,6 +1385,7 @@ def _sanitize_llm_messages(messages: List[Dict]) -> List[Dict]:
 
     return merged
 
+
 def _normalize_anthropic_url(url: str) -> str:
     """Ensure Anthropic URL points to /v1/messages."""
     url = url.rstrip("/")
@@ -1335,8 +1497,10 @@ def list_model_ids(
         r = httpx_get_kimi_aware(models_url, h, timeout=timeout)
         r.raise_for_status()
         data = r.json()
-        model_ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
-        if not model_ids:
+        # Some OpenAI-compatible APIs (e.g. Together) return a bare list here.
+        items = data if isinstance(data, list) else (data.get("data") or [])
+        model_ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
+        if not model_ids and isinstance(data, dict):
             model_ids = [
                 m.get("name") or m.get("model")
                 for m in (data.get("models") or [])
@@ -1424,7 +1588,7 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
             stream=False, num_ctx=get_context_length(url, model),
         )
     else:
-        target_url = url
+        target_url = _normalize_openai_chat_url(url)
         if provider == "copilot":
             from src.copilot import apply_request_headers
             apply_request_headers(h, messages_copy)
@@ -1438,6 +1602,8 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
         if max_tokens and max_tokens > 0:
             tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
             payload[tok_key] = max_tokens
+        if provider == "mistral" and _supports_thinking(model):
+            payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
     try:
         note_model_activity(target_url, model)
         r = httpx_post_kimi_aware(target_url, h, json=payload, timeout=timeout)
@@ -1453,7 +1619,16 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
             response = _parse_ollama_response(data)
         else:
             msg = data["choices"][0]["message"]
-            response = msg.get("content") or msg.get("reasoning_content") or ""
+            content = msg.get("content")
+            if isinstance(content, list):
+                # Mistral structured content — extract thinking + text
+                text_part, thinking_part = _normalize_mistral_content(content)
+                if thinking_part:
+                    response = thinking_part + "\n\n" + (text_part or "")
+                else:
+                    response = text_part or msg.get("reasoning_content") or ""
+            else:
+                response = content or msg.get("reasoning_content") or ""
         _set_cached_response(cache_key, response)
         return response
     except Exception:
@@ -1617,7 +1792,7 @@ async def llm_call_async(
             stream=False, num_ctx=get_context_length(url, model),
         )
     else:
-        target_url = url
+        target_url = _normalize_openai_chat_url(url)
         h = _provider_headers(provider, headers)
         if provider == "copilot":
             from src.copilot import apply_request_headers
@@ -1635,6 +1810,8 @@ async def llm_call_async(
         # Suppress thinking for qwen3/gemma4 on Ollama /v1 — same as stream_llm.
         if _is_ollama_openai_compat_url(url) and _supports_thinking(model):
             payload["think"] = False
+        if provider == "mistral" and _supports_thinking(model):
+            payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
         _apply_local_cache_affinity(payload, url, session_id)
 
     if _is_host_dead(target_url):
@@ -1693,7 +1870,8 @@ async def llm_call_async(
 async def stream_llm(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
                      max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
                      timeout: int = LLMConfig.STREAM_TIMEOUT, prompt_type: Optional[str] = None,
-                     tools: Optional[List[Dict]] = None, session_id: Optional[str] = None):
+                     tools: Optional[List[Dict]] = None, session_id: Optional[str] = None,
+                     tool_choice_none: bool = False):
     """Stream LLM responses with improved error handling.
 
     Yields SSE chunks:
@@ -1737,7 +1915,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
         h = _provider_headers(provider, headers)
         payload = _build_chatgpt_responses_payload(model, messages_copy, temperature, max_tokens, stream=True)
     else:
-        target_url = url
+        target_url = _normalize_openai_chat_url(url)
         payload = {
             "model": model,
             "messages": messages_copy,
@@ -1753,6 +1931,14 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
             payload[tok_key] = max_tokens
         if tools:
             payload["tools"] = tools
+        elif tool_choice_none:
+            payload["tool_choice"] = "none"
+        # Mistral thinking-capable models — send reasoning_effort so Mistral
+        # activates thinking mode and returns structured reasoning_content.
+        # Effort level is configurable via ODYSSEUS_MISTRAL_REASONING_EFFORT
+        # (high / medium / low / none); default "high".
+        if provider == "mistral" and _supports_thinking(model):
+            payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
         # For Ollama's OpenAI-compat /v1 endpoint with thinking models (qwen3,
         # gemma4, etc.), suppress thinking so tool calls aren't swallowed inside
         # <think> blocks. Ollama /v1 accepts "think": false as a top-level param.
@@ -2131,10 +2317,23 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
                                         # Text content
                                         # Reasoning tokens (VLLM --reasoning-parser, e.g. Qwen3/DeepSeek-R1, Nemotron). vLLM 0.20.2 / NIM emit the field as `reasoning`; older builds use `reasoning_content`. Some OpenAI-compatible Ollama builds use `thinking`.
                                         reasoning = delta.get("reasoning_content") or delta.get("reasoning") or delta.get("thinking") or ""
+                                        content = delta.get("content") or ""
+                                        # Mistral structured content: content is a list of typed blocks
+                                        # ({"type": "thinking", ...}, {"type": "text", ...}). Split into
+                                        # reasoning + text so thinking streams into the thinking panel.
+                                        if isinstance(content, list):
+                                            text_part, thinking_part = _normalize_mistral_content(content)
+                                            if thinking_part:
+                                                reasoning = (reasoning + thinking_part) if reasoning else thinking_part
+                                            content = text_part
                                         if reasoning:
                                             yield _stream_delta_event(reasoning, thinking=True)
-                                        content = delta.get("content") or ""
                                         if content:
+                                            content = _strip_visible_chat_template_artifacts(content)
+                                            if not content:
+                                                continue
+                                            content = re.sub(r"<mm:think(\s+[^>]*)?>", r"<think\1>", content, flags=re.IGNORECASE)
+                                            content = re.sub(r"</mm:think>", "</think>", content, flags=re.IGNORECASE)
                                             stripped = content.lstrip()
                                             # gpt-oss harmony format (<|channel|>analysis/final): route via the harmony
                                             # stream router. Sticky once the first marker appears — distinct from the

src/model_context.py

@@ -220,6 +220,10 @@ KNOWN_CONTEXT_WINDOWS = {
     'hermes': 131072,
     'nous-hermes': 131072,
 
+    # --- Xiaomi ---
+    'mimo-v2.5-pro': 1048576,
+    'mimo-v2.5': 1048576,
+
     # --- Open community ---
     'dolphin': 32768,
     'mythomax': 4096,
@@ -312,6 +316,83 @@ def _lookup_known(model: str) -> Optional[int]:
     return best_ctx
 
 
+def _model_ctx_from_entry(m: dict) -> Optional[int]:
+    """Extract a positive context window from one /models catalog entry.
+
+    Checks the common top-level fields first, then a nested meta/model_extra
+    object. Returns None when no positive window is reported.
+    """
+    if not isinstance(m, dict):
+        return None
+    for field in (
+        "context_length",
+        "context_window",
+        "max_model_len",
+        "max_context_length",
+        "max_seq_len",
+    ):
+        val = m.get(field)
+        if val and isinstance(val, (int, float)) and val > 0:
+            return int(val)
+    meta = m.get("meta") or m.get("model_extra") or {}
+    if isinstance(meta, dict):
+        # n_ctx is the actual serving context (set via -c flag in llama.cpp)
+        for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
+            val = meta.get(field)
+            if val and isinstance(val, (int, float)) and val > 0:
+                return int(val)
+    return None
+
+
+# Per-endpoint cache of the {model_id: context_length} map parsed from a
+# proxy/api catalog. api/proxy endpoints skip the /models download on every
+# lookup because a large catalog is expensive; caching the whole map lets us
+# pay that download at most once per endpoint instead of once per model.
+_catalog_ctx_cache: Dict[str, Dict[str, int]] = {}
+
+
+def _proxy_catalog_context(endpoint_url: str, model: str) -> Optional[int]:
+    """Context window for a model read from the endpoint's /models catalog.
+
+    Fetches the catalog once per endpoint and caches the full id->context map,
+    so an api/proxy endpoint serving a model that isn't in KNOWN_CONTEXT_WINDOWS
+    (e.g. a new OpenRouter model) still reports its real window instead of the
+    bare default. Returns None when the catalog can't be read or doesn't list a
+    positive window for the model.
+    """
+    cat = _catalog_ctx_cache.get(endpoint_url)
+    if cat is None:
+        from src.endpoint_resolver import build_models_url
+        try:
+            r = httpx.get(build_models_url(endpoint_url), timeout=REQUEST_TIMEOUT)
+        except Exception as e:
+            logger.debug(f"Failed to fetch proxy catalog for context length: {e}")
+            return None
+        if not r.is_success:
+            return None
+        cat = {}
+        try:
+            for m in (r.json().get("data") or []):
+                mid = m.get("id") if isinstance(m, dict) else None
+                ctx = _model_ctx_from_entry(m) if mid else None
+                if mid and ctx:
+                    cat[mid] = ctx
+        except Exception as e:
+            logger.debug(f"Failed to parse proxy catalog for context length: {e}")
+            return None
+        _catalog_ctx_cache[endpoint_url] = cat
+
+    if model in cat:
+        return cat[model]
+    # Catalog ids may carry a provider prefix (e.g. "openai/gpt-4o") while the
+    # session stores the bare id; match on the trailing segment as a fallback.
+    base = model.split("/")[-1]
+    for mid, ctx in cat.items():
+        if mid.split("/")[-1] == base:
+            return ctx
+    return None
+
+
 def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
     """Query the model API for context length. Returns (context_length, known) where
     ``known`` is False only for the bare DEFAULT_CONTEXT fallback."""
@@ -326,6 +407,14 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
         if known:
             logger.info(f"Using known context window for {model}: {known}")
             return known, True
+        # Not in the known table: read the real window from the catalog (cached
+        # once per endpoint) instead of capping every unknown model at the
+        # default — that under-reported large windows on aggregators like
+        # OpenRouter (issue #4886).
+        api_ctx = _proxy_catalog_context(endpoint_url, model)
+        if api_ctx:
+            logger.info(f"Proxy catalog reports context window for {model}: {api_ctx}")
+            return api_ctx, True
         return DEFAULT_CONTEXT, False
 
     # Try llama.cpp /slots endpoint first — reports actual serving context
@@ -366,27 +455,7 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
             for m in models_list:
                 mid = m.get("id", "")
                 if mid == model or mid.split("/")[-1] == model.split("/")[-1]:
-                    for field in (
-                        "context_length",
-                        "context_window",
-                        "max_model_len",
-                        "max_context_length",
-                        "max_seq_len",
-                    ):
-                        val = m.get(field)
-                        if val and isinstance(val, (int, float)) and val > 0:
-                            api_ctx = int(val)
-                            break
-
-                    if not api_ctx:
-                        meta = m.get("meta") or m.get("model_extra") or {}
-                        if isinstance(meta, dict):
-                            # n_ctx is the actual serving context (set via -c flag in llama.cpp)
-                            for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
-                                val = meta.get(field)
-                                if val and isinstance(val, (int, float)) and val > 0:
-                                    api_ctx = int(val)
-                                    break
+                    api_ctx = _model_ctx_from_entry(m)
                     break
     except Exception as e:
         logger.debug(f"Failed to query context length for {model}: {e}")

src/model_discovery.py

@@ -163,6 +163,21 @@ class ModelDiscovery:
                     return "lmstudio"
         except Exception:
             pass
+        # llama.cpp's llama-server exposes a native /props endpoint (no /v1 prefix)
+        # describing the loaded model, slots, and chat template — distinct from
+        # LM Studio (/api/v1/models) and vLLM (/version, /metrics).
+        try:
+            r = httpx.get(f"http://{host}:{port}/props", timeout=1.5)
+            if r.is_success:
+                props = r.json() or {}
+                if isinstance(props, dict) and (
+                    "default_generation_settings" in props
+                    or "total_slots" in props
+                    or "chat_template" in props
+                ):
+                    return "llamacpp"
+        except Exception:
+            pass
         return None
 
     def _check_port(self, host: str, port: int) -> Optional[Dict[str, Any]]:
@@ -172,8 +187,10 @@ class ModelDiscovery:
             r = httpx.get(f"{base}/models", timeout=3)
             if not r.is_success:
                 return None
-            data = r.json() or {}
-            ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
+            data = r.json()
+            # Some OpenAI-compatible servers return a bare list, not {"data": [...]}.
+            items = data if isinstance(data, list) else ((data or {}).get("data") or [])
+            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
             if ids:
                 return {
                     "host": host,
@@ -194,10 +211,11 @@ class ModelDiscovery:
 
         logger.info(f"Scanning {len(hosts)} hosts for models: {hosts}")
 
-        # Well-known ports: 8000-8020 (vLLM, llama.cpp, SGLang, Cookbook),
-        # 1234 (LM Studio), 11434 (Ollama), 11435 for APFEL as its default port is
-        # occupied by Ollama. The env vars can add more ports which will be merged in.
-        ports = list(range(8000, 8021)) + [1234, 11434, 11435]
+        # Well-known ports: 8000-8020 (vLLM, SGLang, Cookbook), 8080 (llama.cpp /
+        # llama-server default), 1234 (LM Studio), 11434 (Ollama), 11435 for APFEL
+        # as its default port is occupied by Ollama. The env vars can add more
+        # ports which will be merged in.
+        ports = list(range(8000, 8021)) + [8080, 1234, 11434, 11435]
         ports += [p for p in sorted(self._extra_ports) if p not in ports]
         targets = [(h, p) for h in hosts for p in ports]
 

src/personal_docs.py

@@ -68,6 +68,8 @@ def read_text_file(path: str) -> str:
 
 def split_chunks(text: str, size: int = config.CHUNK_SIZE, overlap: int = config.CHUNK_OVERLAP) -> List[str]:
     """Split text into overlapping chunks."""
+    if not isinstance(text, str):
+        return []
     text = text.strip()
     if not text:
         return []
@@ -87,7 +89,8 @@ def split_chunks(text: str, size: int = config.CHUNK_SIZE, overlap: int = config
 
 def tokenize(s: str) -> Set[str]:
     """Tokenize string into words, excluding stop words."""
-    tokens = re.findall(r"[A-Za-z0-9_\-]+", (s or "").lower())
+    text = s if isinstance(s, str) else ""
+    tokens = re.findall(r"[A-Za-z0-9_\-]+", text.lower())
     return set(t for t in tokens if t not in config.STOP_WORDS and len(t) > 1)
 
 def load_personal_index(