ci: add focused test guidance signal (#4982)

* ci: add focused test guidance signal

* ci: diff focused guidance from merge base

.env.example

@@ -169,6 +169,26 @@ SEARXNG_INSTANCE=http://localhost:8080
 # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400                  # speech-to-text audio (25 MB)
 # ODYSSEUS_ICS_MAX_BYTES=10485760                        # calendar .ics import (10 MB)
 
+# ============================================================
+# Host Docker access (explicit opt-in)
+# ============================================================
+# Default Docker Compose does not mount /var/run/docker.sock. Existing
+# Ollama, vLLM, and other OpenAI-compatible endpoints remain usable without it.
+#
+# Enable this only for intentional Cookbook/local Docker-daemon management.
+# Raw socket access is high-trust and can grant broad control over the host
+# Docker daemon. Set DOCKER_GID to the host docker group's numeric GID.
+# Put these values in .env, or export them before running docker compose.
+# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
+# DOCKER_GID=963
+# docker/host-docker.yml sets this inside the container. Keep it paired
+# with the socket overlay; setting it alone is not sufficient.
+# ODYSSEUS_ENABLE_HOST_DOCKER=true
+#
+# Host Docker access can be combined with one GPU overlay:
+# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
+# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
+
 # ============================================================
 # GPU support (Docker Compose)
 # ============================================================

.github/scripts/focused_test_guidance.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Report focused pytest guidance for changed paths under tests/."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+from collections.abc import Iterable
+from pathlib import PurePosixPath
+
+
+def parse_paths(raw_paths: bytes) -> list[str]:
+    """Decode the NUL-delimited output of ``git diff --name-only -z``."""
+    return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path]
+
+
+def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]:
+    """Return changed ``tests/`` paths using GitHub PR three-dot semantics.
+
+    GitHub PR changed files are based on the merge base and the PR head, not a
+    direct endpoint diff between the current base branch tip and the PR head.
+    Using the direct endpoint diff can include files changed only on the base
+    branch when the PR branch is stale.
+    """
+    merge_base = subprocess.check_output(
+        ["git", "merge-base", base_sha, head_sha],
+        stderr=subprocess.DEVNULL,
+    ).strip()
+    raw_paths = subprocess.check_output(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "--diff-filter=ACMRT",
+            "-z",
+            os.fsdecode(merge_base),
+            head_sha,
+            "--",
+            "tests/",
+        ],
+    )
+    return parse_paths(raw_paths)
+
+
+def select_test_paths(paths: Iterable[str]) -> list[str]:
+    """Return unique, repository-relative paths contained by tests/."""
+    selected: set[str] = set()
+    for raw_path in paths:
+        path = PurePosixPath(raw_path)
+        if path.is_absolute() or ".." in path.parts:
+            continue
+        parts = tuple(part for part in path.parts if part != ".")
+        if len(parts) >= 2 and parts[0] == "tests":
+            selected.add(PurePosixPath(*parts).as_posix())
+    return sorted(selected)
+
+
+def is_pytest_file(path: str) -> bool:
+    """Return whether a changed path follows this repository's pytest naming."""
+    name = PurePosixPath(path).name
+    return name.endswith(".py") and (
+        name.startswith("test_") or name.endswith("_test.py")
+    )
+
+
+def pytest_command(paths: Iterable[str]) -> str:
+    """Build a copyable pytest command for changed runnable test files."""
+    command = ["python3", "-m", "pytest", "-q", *paths]
+    return shlex.join(command)
+
+
+def format_report(paths: Iterable[str]) -> str:
+    """Format focused guidance for CI logs and the workflow summary."""
+    changed_paths = select_test_paths(paths)
+    runnable_paths = [path for path in changed_paths if is_pytest_file(path)]
+    lines = ["## Focused test guidance (report-only)", ""]
+    if not changed_paths:
+        lines.append("No changed paths under `tests/`.")
+    else:
+        lines.extend(["Changed paths under `tests/`:", ""])
+        lines.extend(f"- `{path}`" for path in changed_paths)
+    lines.extend(["", "Suggested focused validation:", ""])
+    if runnable_paths:
+        lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```")
+    else:
+        lines.append("No directly runnable pytest files changed.")
+    lines.extend(
+        [
+            "",
+            "This guidance does not infer tests from source changes. "
+            "Existing blocking CI remains the source of truth.",
+        ]
+    )
+    return "\n".join(lines)
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Report focused pytest guidance for changed tests/ paths.",
+    )
+    parser.add_argument("--base-sha", help="Pull request base commit SHA.")
+    parser.add_argument("--head-sha", help="Pull request head commit SHA.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(sys.argv[1:] if argv is None else argv)
+    if bool(args.base_sha) != bool(args.head_sha):
+        raise SystemExit("--base-sha and --head-sha must be provided together")
+
+    if args.base_sha and args.head_sha:
+        paths = changed_paths_from_merge_base(args.base_sha, args.head_sha)
+    else:
+        paths = parse_paths(sys.stdin.buffer.read())
+
+    print(format_report(paths))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/ci.yml

@@ -15,6 +15,60 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  focused-test-guidance:
+    name: Focused test guidance (report-only)
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Report changed test paths
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          report_file="$RUNNER_TEMP/focused-test-guidance.md"
+          publish_report() {
+            cat "$report_file"
+            if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+              cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true
+            fi
+            return 0
+          }
+
+          report_unavailable() {
+            {
+              printf '%s\n\n' '## Focused test guidance unavailable (report-only)'
+              printf '%s\n\n' "$1"
+              printf '%s\n' 'Existing blocking CI remains the source of truth.'
+            } > "$report_file"
+            publish_report
+            exit 0
+          }
+
+          if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
+            report_unavailable "Pull request base/head metadata is missing."
+          fi
+
+          if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
+            report_unavailable "The pull request base commit is unavailable locally."
+          fi
+
+          if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then
+            report_unavailable "The pull request head commit is unavailable locally."
+          fi
+
+          if ! python3 .github/scripts/focused_test_guidance.py \
+            --base-sha "$BASE_SHA" \
+            --head-sha "$HEAD_SHA" > "$report_file"; then
+            report_unavailable "The focused test guidance helper could not produce a report."
+          fi
+
+          publish_report
+
   python-syntax:
     name: Python syntax (compileall)
     runs-on: ubuntu-latest

app.py

@@ -197,7 +197,19 @@ class _RequestTimeoutMiddleware(_BaseHTTPMiddleware):
             )
 
 
+class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
+    async def dispatch(self, request, call_next):
+        from src.interactive_gate import should_track_interactive_request, track_interactive_request
+
+        path = request.url.path or ""
+        if not should_track_interactive_request(path, request.method):
+            return await call_next(request)
+        async with track_interactive_request(path, request.method):
+            return await call_next(request)
+
+
 app.add_middleware(_RequestTimeoutMiddleware)
+app.add_middleware(_InteractiveActivityMiddleware)
 
 # ========= AUTH =========
 from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
@@ -583,6 +595,14 @@ webhook_manager = WebhookManager(api_key_manager=api_key_manager)
 auth_router = setup_auth_routes(auth_manager)
 app.include_router(auth_router)
 
+
+@app.post("/api/activity/heartbeat")
+async def activity_heartbeat():
+    from src.interactive_gate import mark_browser_activity
+    await mark_browser_activity()
+    return {"ok": True}
+
+
 # Uploads
 from routes.upload_routes import setup_upload_routes
 upload_router, upload_cleanup_func = setup_upload_routes(upload_handler)
@@ -604,7 +624,7 @@ from routes.admin_wipe_routes import setup_admin_wipe_routes
 app.include_router(setup_admin_wipe_routes(session_manager))
 
 # Memory
-from routes.memory_routes import setup_memory_routes
+from routes.memory.memory_routes import setup_memory_routes
 memory_router = setup_memory_routes(memory_manager, session_manager, memory_vector=memory_vector)
 app.include_router(memory_router)
 from routes.skills_routes import setup_skills_routes
@@ -621,7 +641,7 @@ app.include_router(setup_chat_routes(
 ))
 
 # Research (background deep-research tasks)
-from routes.research_routes import setup_research_routes
+from routes.research.research_routes import setup_research_routes
 app.include_router(setup_research_routes(research_handler, session_manager=session_manager))
 
 # History
@@ -1005,17 +1025,21 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
 
-    # Keep-alive: ping endpoints every 60 seconds to prevent cold starts
-    async def _keepalive_loop():
-        while True:
-            try:
-                await asyncio.sleep(60)
-                await _warmup_endpoints()
-            except Exception as e:
-                logger.warning(f"Keepalive loop error: {e}")
-                await asyncio.sleep(300)  # Back off on error
+    # Keep-alive is opt-in. The ping path performs model discovery, and when
+    # stale LAN endpoints are configured it can add periodic backend pressure
+    # that delays unrelated UI requests such as Notes/Documents.
+    _keepalive_enabled = str(os.getenv("ODYSSEUS_MODEL_KEEPALIVE", "")).lower() in {"1", "true", "yes", "on"}
+    if _keepalive_enabled:
+        async def _keepalive_loop():
+            while True:
+                try:
+                    await asyncio.sleep(60)
+                    await _warmup_endpoints()
+                except Exception as e:
+                    logger.warning(f"Keepalive loop error: {e}")
+                    await asyncio.sleep(300)  # Back off on error
 
-    _startup_tasks.append(asyncio.create_task(_keepalive_loop()))
+        _startup_tasks.append(asyncio.create_task(_keepalive_loop()))
 
     async def _ensure_default_tasks():
         # Create/reconcile default automation tasks + personal assistant for every user.

core/atomic_io.py

@@ -34,6 +34,8 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) ->
 
 
 def atomic_write_text(path: str, text: str) -> None:
+    if not isinstance(text, str):
+        raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
     tmp = f"{path}.tmp.{os.getpid()}"
     with open(tmp, "w", encoding="utf-8") as f:

core/database.py

@@ -276,6 +276,7 @@ class GalleryImage(TimestampMixin, Base):
     id         = Column(String, primary_key=True, index=True)
     filename   = Column(String, nullable=False, unique=True)
     prompt     = Column(Text, nullable=False, default="")
+    caption    = Column(Text, nullable=True, default="")
     model      = Column(String, nullable=True)
     size       = Column(String, nullable=True)
     quality    = Column(String, nullable=True)
@@ -1182,6 +1183,29 @@ def _migrate_add_multiuser_owner_columns():
     _migrate_add_owner_to_table("documents", "ix_documents_owner")
 
 
+def _migrate_add_gallery_caption_column():
+    """Add OCR/vision caption storage for gallery images."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(gallery_images)").fetchall()]
+        if columns and "caption" not in columns:
+            conn.execute("ALTER TABLE gallery_images ADD COLUMN caption TEXT DEFAULT ''")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added caption column to gallery_images")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration gallery caption column failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+
 def _migrate_add_api_token_scopes_column():
     """Add API token scopes for existing installs.
 
@@ -1670,6 +1694,7 @@ class CalendarEvent(TimestampMixin, Base):
     # `Z`-suffix on serialization so the frontend interprets correctly.
     is_utc      = Column(Boolean, default=False, nullable=False)
     rrule       = Column(String, default="")
+    recurrence_exdates = Column(Text, default="")  # JSON list of skipped occurrence starts
     color       = Column(String, nullable=True)  # per-event color override
     status      = Column(String, default="confirmed")  # confirmed, cancelled
     importance  = Column(String, default="normal")    # low | normal | high | critical
@@ -1811,6 +1836,7 @@ def init_db():
     _migrate_add_token_columns()
     _migrate_add_mode_column()
     _migrate_add_multiuser_owner_columns()
+    _migrate_add_gallery_caption_column()
     _migrate_add_api_token_scopes_column()
     _migrate_backfill_document_owner_from_session()
     _migrate_assign_legacy_owner()
@@ -1833,6 +1859,7 @@ def init_db():
     _migrate_add_calendar_origin()
     _migrate_add_calendar_account_id()
     _migrate_add_caldav_sync_columns()
+    _migrate_add_calendar_recurrence_exdates()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2184,6 +2211,28 @@ def _migrate_add_calendar_metadata():
         except Exception:
             pass
 
+
+def _migrate_add_calendar_recurrence_exdates():
+    """Add skipped recurrence occurrences for deleting one instance of a series."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
+        if columns and "recurrence_exdates" not in columns:
+            conn.execute("ALTER TABLE calendar_events ADD COLUMN recurrence_exdates TEXT DEFAULT ''")
+        conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"calendar_events recurrence_exdates migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def get_db():
     """
     Dependency to get a database session.

core/middleware.py

@@ -117,7 +117,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
                 f"script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net; "
                 "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
                 "font-src 'self' https://cdn.jsdelivr.net; "
-                "img-src 'self' data: blob:; "
+                "img-src 'self' data: blob: https:; "
                 "media-src 'self' blob:; "
                 "connect-src 'self'; "
                 "frame-src 'self'; "

docker-compose.gpu-amd.yml

@@ -28,14 +28,6 @@ services:
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
       - ${APP_DATA_DIR:-./data}/local:/app/.local:z
-      # Docker socket — lets Cookbook launch commands like
-      # `docker exec ollama-rocm ollama show <tag>` reach the host's
-      # Docker daemon (and sibling containers like ollama-rocm /
-      # ollama-test). The in-container user needs to be in the
-      # socket's owning group — see `group_add` below; the GID
-      # there must match the host's `docker` group (defaults to 963
-      # on Debian, 999 on Ubuntu — override via env if yours differs).
-      - /var/run/docker.sock:/var/run/docker.sock
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.
@@ -101,7 +93,6 @@ services:
       - /dev/kfd
       - /dev/dri
     group_add:
-      - "${DOCKER_GID:-963}"
       - video
       - ${RENDER_GID:-render}
 

docker-compose.gpu-nvidia.yml

@@ -27,16 +27,6 @@ services:
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
       - ${APP_DATA_DIR:-./data}/local:/app/.local:z
-      # Docker socket — lets Cookbook launch commands like
-      # `docker exec ollama-rocm ollama show <tag>` reach the host's
-      # Docker daemon (and sibling containers like ollama-rocm /
-      # ollama-test). The in-container user needs to be in the
-      # socket's owning group — see `group_add` below; the GID
-      # there must match the host's `docker` group (defaults to 963
-      # on Debian, 999 on Ubuntu — override via env if yours differs).
-      - /var/run/docker.sock:/var/run/docker.sock
-    group_add:
-      - "${DOCKER_GID:-963}"
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.

docker-compose.yml

@@ -16,16 +16,6 @@ services:
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
       - ${APP_DATA_DIR:-./data}/local:/app/.local:z
-      # Docker socket — lets Cookbook launch commands like
-      # `docker exec ollama-rocm ollama show <tag>` reach the host's
-      # Docker daemon (and sibling containers like ollama-rocm /
-      # ollama-test). The in-container user needs to be in the
-      # socket's owning group — see `group_add` below; the GID
-      # there must match the host's `docker` group (defaults to 963
-      # on Debian, 999 on Ubuntu — override via env if yours differs).
-      - /var/run/docker.sock:/var/run/docker.sock
-    group_add:
-      - "${DOCKER_GID:-963}"
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.

docker/entrypoint.sh

@@ -29,12 +29,12 @@ fi
 ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)"
 [ -z "$ODY_USER" ] && ODY_USER=odysseus
 
-# Docker-socket group plumbing. When /var/run/docker.sock is bind-mounted
-# (Cookbook uses docker exec to reach sibling containers), the socket is
-# owned by root:<host docker gid>. Add the app user to that group and later
-# call gosu by username so supplementary groups are retained.
+# Docker-socket group plumbing for the explicit host-Docker overlay. When
+# opted in, the socket is owned by root:<host docker gid>. Add the app user
+# to that group and later call gosu by username so supplementary groups are
+# retained.
 DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}"
-if [ -S "$DOCKER_SOCK" ]; then
+if [ "${ODYSSEUS_ENABLE_HOST_DOCKER:-}" = "true" ] && [ -S "$DOCKER_SOCK" ]; then
     SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')"
     if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then
         if ! getent group "$SOCK_GID" >/dev/null 2>&1; then

docker/host-docker.yml

@@ -0,0 +1,12 @@
+# High-trust host Docker access. Enable only when local Docker-daemon
+# management from Cookbook is required and you accept that raw socket access
+# grants broad control over the host Docker daemon.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
+#   DOCKER_GID=<numeric host Docker group id>
+services:
+  odysseus:
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    group_add: ["${DOCKER_GID:-963}"]
+    environment:
+      - ODYSSEUS_ENABLE_HOST_DOCKER=true

docs/setup.md

@@ -99,6 +99,33 @@ Odysseus SSH key and add the public key to the remote server's
 ssh-copy-id -i data/ssh/id_ed25519.pub user@server
 ```
 
+**Host Docker access (explicit opt-in).** Default Docker Compose intentionally
+does not mount `/var/run/docker.sock`. You can still connect Odysseus to
+existing Ollama, vLLM, and other OpenAI-compatible endpoints without Docker
+socket access.
+
+Cookbook/local Docker-daemon management requires the opt-in overlay below. Raw
+Docker socket access is high-trust because it can effectively grant broad
+control over the host Docker daemon. Remote server Docker workflows over SSH
+remain preferred.
+
+Place these values in `.env`, or export them in the shell before running
+`docker compose`:
+
+```bash
+COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
+DOCKER_GID=<host docker group gid>
+```
+
+Combine host Docker access with a GPU overlay when both are intentionally
+required:
+
+```bash
+COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
+# or
+COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
+```
+
 **Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
 only detect GPUs that Docker exposes to the container — if the host runtime or
 device passthrough is not configured, Cookbook sees the iGPU, another card, or

mcp_servers/email_server.py

@@ -538,6 +538,148 @@ def _get_cached_summaries():
         return {}
 
 
+def _fixture_email_file() -> Path:
+    return DATA_DIR / "fixture_email_messages.json"
+
+
+def _fixture_email_enabled() -> bool:
+    return _fixture_email_file().exists()
+
+
+def _parse_fixture_date(raw_date: str) -> tuple[str, float]:
+    if not raw_date:
+        return "", 0.0
+    parsed = None
+    try:
+        parsed = datetime.fromisoformat(str(raw_date).replace("Z", "+00:00"))
+    except Exception:
+        try:
+            parsed = email.utils.parsedate_to_datetime(str(raw_date))
+        except Exception:
+            parsed = None
+    if parsed:
+        return parsed.isoformat(), parsed.timestamp()
+    return str(raw_date), 0.0
+
+
+def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict:
+    sender = str(row.get("from") or "Fixture Sender <fixture@example.invalid>")
+    sender_name, sender_addr = email.utils.parseaddr(sender)
+    date_str, date_epoch = _parse_fixture_date(str(row.get("date") or ""))
+    subject = str(row.get("subject") or "(no subject)")
+    body = str(row.get("body") or "")
+    owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default")
+    uid = str(uid_num)
+    return {
+        "uid": uid,
+        "message_id": f"<fixture-email-{uid}-{owner_key}@fixtures.odysseus.local>",
+        "subject": subject,
+        "from": sender_name or sender_addr or sender,
+        "from_address": sender_addr,
+        "date": date_str,
+        "date_epoch": date_epoch,
+        "summary": body[:240],
+        "body": body,
+        "account": "Fixture Inbox",
+        "account_email": owner or str(row.get("owner") or ""),
+        "account_id": "fixture-email",
+        "attachments": [],
+    }
+
+
+def _fixture_email_rows(owner: str | None = None) -> list[dict]:
+    path = _fixture_email_file()
+    if not path.exists():
+        return []
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return []
+    rows = raw.get("messages") if isinstance(raw, dict) else raw
+    out = []
+    owner = str(owner or "").strip()
+    for i, row in enumerate(rows if isinstance(rows, list) else [], start=1):
+        if not isinstance(row, dict):
+            continue
+        row_owner = str(row.get("owner") or "").strip()
+        if owner and row_owner and row_owner != owner:
+            continue
+        out.append(_fixture_email_record(row, i, owner or row_owner))
+    out.sort(key=lambda item: item.get("date_epoch") or 0, reverse=True)
+    return out
+
+
+def _fixture_account_rows() -> list[dict]:
+    if not _fixture_email_enabled():
+        return []
+    owner = _current_owner()
+    owners = []
+    for row in _fixture_email_rows(owner or None):
+        email_addr = row.get("account_email") or owner or "fixture@fixtures.odysseus.local"
+        if email_addr not in owners:
+            owners.append(email_addr)
+    if not owners:
+        owners = [owner or "fixture@fixtures.odysseus.local"]
+    return [
+        {
+            "id": "fixture-email",
+            "owner": owner or owners[0],
+            "name": "Fixture Inbox",
+            "is_default": True,
+            "imap_user": owners[0],
+            "from_address": owners[0],
+        }
+    ]
+
+
+def _fixture_email_matches(item: dict, query: str) -> bool:
+    if not query:
+        return True
+    terms = [term for term in re.split(r"\W+", str(query).lower()) if term]
+    haystack = "\n".join(
+        str(item.get(key) or "")
+        for key in ("subject", "from", "from_address", "body", "summary")
+    ).lower()
+    return all(term in haystack for term in terms)
+
+
+def _fixture_list_emails(folder="INBOX", max_results=20, unresponded_only=False,
+                         unread_only=False, account=None) -> list[dict] | None:
+    if not _fixture_email_enabled():
+        return None
+    if account and str(account).strip().lower() not in {
+        "fixture-email",
+        "fixture inbox",
+        "fixture",
+        str(_current_owner()).lower(),
+    }:
+        return []
+    if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
+        return []
+    return _fixture_email_rows(_current_owner())[: int(max_results or 20)]
+
+
+def _fixture_search_emails(query, folders=None, max_results=20, account=None) -> list[dict] | None:
+    if not _fixture_email_enabled():
+        return None
+    rows = _fixture_list_emails("INBOX", max_results=1000, account=account) or []
+    out = [dict(row, _folder="INBOX") for row in rows if _fixture_email_matches(row, str(query or ""))]
+    return out[: int(max_results or 20)]
+
+
+def _fixture_read_email(uid=None, message_id=None, folder="INBOX", account=None) -> dict | None:
+    if not _fixture_email_enabled():
+        return None
+    if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
+        return {"error": f"Email UID {uid or message_id} not found"}
+    for item in _fixture_email_rows(_current_owner()):
+        if uid and str(item.get("uid")) == str(uid):
+            return item
+        if message_id and str(item.get("message_id")) == str(message_id):
+            return item
+    return {"error": f"Email not found with UID/Message-ID: {uid or message_id}"}
+
+
 # ── Tool implementations ──
 
 
@@ -548,6 +690,9 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False,
     Pass unread_only=True and/or unresponded_only=True for attention scans.
     account selects mailbox (None = default).
     """
+    fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, account)
+    if fixture is not None:
+        return fixture
     conn = None
     try:
         conn = _imap_connect(account)
@@ -629,6 +774,9 @@ def _result_sort_time(result: dict) -> datetime:
 
 def _list_emails_across_accounts(folder="INBOX", max_results=20,
                                  unresponded_only=False, unread_only=False):
+    fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, None)
+    if fixture is not None:
+        return fixture, []
     rows = _list_accounts_raw()
     combined = []
     errors = []
@@ -662,6 +810,9 @@ def _search_emails(query, folders=None, max_results=20, account=None):
     _list_emails plus an `_folder` tag."""
     if not query or not str(query).strip():
         return []
+    fixture = _fixture_search_emails(query, folders=folders, max_results=max_results, account=account)
+    if fixture is not None:
+        return fixture
     q = str(query).replace("\\", "\\\\").replace('"', '\\"')
     # Mail clients commonly use OR FROM/SUBJECT/TEXT to match either field.
     # IMAP SEARCH OR is binary, so we nest it.
@@ -784,6 +935,9 @@ def _extract_attachment_to_disk(msg, index, target_dir):
 
 def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
     """Read full email content by UID or message-ID. account = mailbox selector."""
+    fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=account)
+    if fixture is not None:
+        return fixture
     cfg = _load_config(account)
     conn = None
     try:
@@ -837,6 +991,9 @@ def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
 
 
 def _read_email_across_accounts(uid=None, message_id=None, folder="INBOX"):
+    fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=None)
+    if fixture is not None:
+        return fixture
     rows = _list_accounts_raw()
     matches = []
     errors = []
@@ -1775,9 +1932,10 @@ async def list_tools() -> list[Tool]:
         Tool(
             name="reply_to_email",
             description=(
-                "Reply to an existing email by UID. This sends immediately; for normal "
-                "assistant-written replies, prefer draft_email_reply so the user can "
-                "review and send from Odysseus. Automatically threads the reply with "
+                "Reply to an existing email by UID. This sends immediately. Do NOT use "
+                "for normal 'write/draft a reply saying X' requests; use "
+                "draft_email_reply so the user can review and send from Odysseus. "
+                "Only use this when the user explicitly says to send now. Automatically threads the reply with "
                 "In-Reply-To and References headers, prefixes 'Re:' on the subject, and "
                 "uses the original sender as the recipient. Set reply_all=true to also CC "
                 "the original To/Cc recipients. For follow-up 'reply ...' requests, use "
@@ -1991,6 +2149,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
 
         if name == "list_email_accounts":
             rows = _filter_accounts_for_owner(all_db_accounts)
+            if not rows:
+                rows = _fixture_account_rows()
             if not rows:
                 if all_db_accounts and owner:
                     return [TextContent(type="text", text="No email accounts configured for this owner.")]

requirements.txt

@@ -3,6 +3,7 @@ uvicorn
 python-multipart
 python-dotenv
 httpx
+httpcore>=1.0,<2.0
 pydantic>=2.13.4
 pydantic-settings>=2.14.1
 SQLAlchemy

routes/calendar_routes.py

@@ -1,6 +1,7 @@
 """Calendar routes — local SQLite-backed calendar CRUD."""
 
 import logging
+import json
 import re
 import uuid
 from datetime import datetime, date, timedelta
@@ -541,6 +542,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "description": ev.description or "",
         "location": ev.location or "",
         "rrule": ev.rrule or "",
+        "recurrence_exdates": _recurrence_exdates(ev),
         "calendar": ev.calendar.name if ev.calendar else "",
         "calendar_href": ev.calendar_id,
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
@@ -554,6 +556,28 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
 _RRULE_EXPANSION_LIMIT = 1000
 
 
+def _recurrence_exdates(ev: CalendarEvent) -> list[str]:
+    raw = getattr(ev, "recurrence_exdates", "") or ""
+    if not raw:
+        return []
+    try:
+        values = json.loads(raw)
+    except Exception:
+        return []
+    if not isinstance(values, list):
+        return []
+    return [str(v) for v in values if isinstance(v, str) and v.strip()]
+
+
+def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
+    if "::" not in uid:
+        return ""
+    suffix = uid.split("::", 1)[1]
+    if ev.all_day:
+        return suffix[:10]
+    return suffix[:16]
+
+
 def _expand_rrule(
     ev: CalendarEvent, start: datetime, end: datetime
 ) -> List[dict]:
@@ -618,6 +642,7 @@ def _expand_rrule(
     results = []
     truncated = False
     base = _event_to_dict(ev)
+    exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
         if occ_start >= end:
@@ -638,8 +663,13 @@ def _expand_rrule(
         # Build the compound uid: {base_uid}::{date} or ::{datetime}
         if ev.all_day:
             occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}"
+            exdate_key = occ_start.strftime("%Y-%m-%d")
         else:
             occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%dT%H:%M')}"
+            exdate_key = occ_start.strftime("%Y-%m-%dT%H:%M")
+
+        if exdate_key in exdates:
+            continue
 
         d = dict(base)
         d["uid"] = occ_uid
@@ -1150,7 +1180,7 @@ def setup_calendar_routes() -> APIRouter:
             db.close()
 
     @router.delete("/events/{uid}")
-    async def delete_event(request: Request, uid: str):
+    async def delete_event(request: Request, uid: str, scope: str = "series"):
         owner = _require_user(request)
         try:
             base_uid = _resolve_base_uid(uid)
@@ -1159,7 +1189,22 @@ def setup_calendar_routes() -> APIRouter:
         db = SessionLocal()
         try:
             ev = _get_or_404_event(db, base_uid, owner)
+            is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if is_occurrence_delete:
+                key = _occurrence_exdate_key(uid, ev)
+                if not key:
+                    raise HTTPException(400, "Invalid recurring occurrence uid")
+                exdates = _recurrence_exdates(ev)
+                if key not in exdates:
+                    exdates.append(key)
+                ev.recurrence_exdates = json.dumps(sorted(exdates))
+                if is_caldav:
+                    ev.caldav_sync_pending = "update"
+                db.commit()
+                if is_caldav:
+                    await _push_caldav_event_after_commit(owner, base_uid, "update")
+                return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
             db.delete(ev)

routes/chat_routes.py

@@ -729,6 +729,15 @@ def setup_chat_routes(
                         logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}")
                 else:
                     logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}")
+            if not active_doc:
+                _email_doc_q = _doc_db.query(DBDocument).filter(
+                    DBDocument.session_id == session,
+                    DBDocument.is_active == True,
+                    DBDocument.language == "email",
+                )
+                active_doc = _owner_session_filter(_email_doc_q, ctx.user).order_by(DBDocument.updated_at.desc()).first()
+                if active_doc:
+                    logger.info(f"[doc-inject] found email draft by session fallback: title={active_doc.title!r}")
             if not active_doc:
                 _session_doc_q = _doc_db.query(DBDocument).filter(
                     DBDocument.session_id == session,
@@ -790,19 +799,19 @@ def setup_chat_routes(
                 "manage_skills",      # skill presets tied to user
             })
 
-        # Active email reader open → strip the tools that let the agent
-        # "drift" to a new compose: create_document (writes a fake email-
-        # shaped .md file) and send_email (sends fresh to a recipient the
-        # agent invented). With those gone, the only paths left for "write
-        # email saying X" are ui_control open_email_reply (draft) and
-        # reply_to_email (immediate send) — both of which use the open
-        # email's UID. Code-level enforcement instead of relying on a
-        # prompt rule the model can ignore.
+        # Active email reader open → strip the tools that let the agent drift
+        # away from the visible email or skip review. The only allowed compose
+        # path is ui_control open_email_reply, which opens the same draft editor
+        # as the Reply button with the generated body pre-filled. This prevents
+        # the model from falling back to direct SMTP when it botches a draft
+        # call, and prevents fake email-shaped documents.
         if active_email_ctx and active_email_ctx.get("uid"):
             disabled_tools.update({
                 "create_document",
                 "send_email",
+                "reply_to_email",
                 "mcp__email__send_email",
+                "mcp__email__reply_to_email",
             })
 
         # Enforce per-user privileges
@@ -1350,9 +1359,11 @@ def setup_chat_routes(
                         elif chunk.startswith("event: "):
                             yield chunk
                         elif chunk == "data: [DONE]\n\n":
-                            if full_response:
+                            _has_tool_events = bool((last_metrics or {}).get("tool_events"))
+                            if full_response or _has_tool_events:
+                                _response_to_save = full_response or "Done."
                                 _saved_id = save_assistant_response(
-                                    sess, session_manager, session, full_response, last_metrics,
+                                    sess, session_manager, session, _response_to_save, last_metrics,
                                     character_name=ctx.preset.character_name,
                                     web_sources=web_sources,
                                     rag_sources=ctx.rag_sources,
@@ -1362,7 +1373,7 @@ def setup_chat_routes(
                                 if _saved_id:
                                     yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
                                 run_post_response_tasks(
-                                    sess, session_manager, session, message, full_response,
+                                    sess, session_manager, session, message, _response_to_save,
                                     last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
                                     incognito=incognito, compare_mode=compare_mode,
                                     character_name=ctx.preset.character_name,

routes/cookbook_helpers.py

@@ -558,6 +558,18 @@ def _bash_squote(v: str) -> str:
     return v.replace("'", "'\\''")
 
 
+# Shown by generated runner scripts when the ollama binary is missing on the
+# target host. Must stay free of backticks/$( ) and be emitted single-quoted:
+# an earlier version wrapped the install one-liner in backticks inside a
+# double-quoted echo, which bash executed as command substitution and ran the
+# system-wide installer (including on remote SSH hosts) instead of printing
+# the hint.
+OLLAMA_MISSING_HINT = (
+    "ERROR: Ollama not found on this server. Install it from "
+    "https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh"
+)
+
+
 # Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
 # Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
 _SERVE_CMD_ALLOWLIST = {
@@ -577,6 +589,16 @@ _SERVE_CMD_ALLOWLIST = {
 _GGUF_PRELUDE_RE = re.compile(
     r'^MODEL_FILE=\$\([^\n]*?\)\s*&&\s*\{[^{}]*\}\s*\|\|\s*\{[^{}]*\}\s*&&\s*'
 )
+_SAFE_SUBSHELL_TEXT = r"[^'\n;&|`$()<>]+"
+_SAFE_SUBSHELL_DQ_HOME_PATH = r'"\$HOME/[^"\n;&|`()<>]*"'
+_SAFE_PRINTF_SUBSHELL_RE = re.compile(
+    rf"^\$\(printf[ \t]+%s[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|\$\{{HOME\}}'/{_SAFE_SUBSHELL_TEXT}')\)$"
+)
+_SAFE_FIND_MMPROJ_SUBSHELL_RE = re.compile(
+    rf"^\$\(find[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|{_SAFE_SUBSHELL_DQ_HOME_PATH}|{_SAFE_SUBSHELL_TEXT})"
+    r"[ \t]+-iname[ \t]+'mmproj\*\.gguf'"
+    r"(?:[ \t]+2>/dev/null)?[ \t]*\|[ \t]*sort[ \t]*\|[ \t]*head[ \t]+-1\)$"
+)
 _OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)")
 _OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$")
 _OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
@@ -677,6 +699,13 @@ def _check_serve_binary(seg: str) -> None:
         )
 
 
+def _is_safe_serve_subshell(subshell: str) -> bool:
+    return bool(
+        _SAFE_PRINTF_SUBSHELL_RE.fullmatch(subshell)
+        or _SAFE_FIND_MMPROJ_SUBSHELL_RE.fullmatch(subshell)
+    )
+
+
 def _validate_serve_cmd(v: str | None) -> str | None:
     """Reject serve commands that aren't in the allowlist or contain shell metachars.
 
@@ -708,15 +737,15 @@ def _validate_serve_cmd(v: str | None) -> str | None:
             _check_serve_binary(part.strip())
         return v
 
-    # Otherwise: a single invocation — no shell metacharacters allowed.
-    # Temporarily replace safe $(printf %s ...) expressions with a placeholder
-    # to avoid triggering the metacharacter/command-injection checks.
-    cleaned_v = v
-    printf_matches = list(re.finditer(r"\$\(\s*printf\s+%s\s+([^\n()]*?)\)", v))
-    for match in printf_matches:
-        inner = match.group(1)
-        if not any(c in inner for c in (";", "&&", "||", "$(", "`")):
-            cleaned_v = cleaned_v.replace(match.group(0), "/placeholder/safe/path.gguf")
+    # Otherwise: a single invocation — no shell metacharacters allowed. Replace
+    # only the exact command substitutions emitted by the Cookbook UI:
+    # $(printf %s 'safe-path') and the mmproj lookup
+    # $(find <path> -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1).
+    def _replace_safe_subshell(match: re.Match[str]) -> str:
+        subshell = match.group(0)
+        return "/placeholder/safe/path" if _is_safe_serve_subshell(subshell) else subshell
+
+    cleaned_v = re.sub(r"\$\([^()]*\)", _replace_safe_subshell, v)
 
     # (`$(` was the original intent; bare `$` is fine for shell-safe paths.)
     if any(c in cleaned_v for c in (";", "&&", "||", "$(")):

routes/cookbook_routes.py

@@ -9,6 +9,8 @@ import shlex
 import shutil
 import subprocess
 import sys
+import time
+import urllib.request
 import uuid
 from pathlib import Path
 
@@ -30,6 +32,13 @@ from core.platform_compat import (
     which_tool,
 )
 from routes.shell_routes import TMUX_LOG_DIR
+from src.host_docker_access import (
+    HOST_DOCKER_ACCESS_HINT,
+    HOST_DOCKER_SOCKET_PATH,
+    host_docker_access_enabled,
+    local_docker_available,
+    running_in_container,
+)
 from routes.cookbook_output import (
     error_aware_output_tail, classify_dead_download,
     HF_CACHE_COMPLETE_PROBE, HF_CACHE_INCOMPLETE_PROBE,
@@ -40,7 +49,7 @@ logger = logging.getLogger(__name__)
 from routes.cookbook_helpers import (
     _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
     _validate_local_dir, _validate_gpus, _shell_path,
-    _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase,
+    _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
     _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
     _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
     load_stored_hf_token,
@@ -62,9 +71,188 @@ _HF_TOKEN_STATUS_SNIPPET = (
     'fi'
 )
 
+_OLLAMA_SIDECAR_CONTAINERS = {"ollama-test", "ollama-rocm"}
+_UNSAFE_DOCKER_EXEC_CHARS = frozenset(";&|<>$`\r\n")
+_SAFE_OLLAMA_MODEL_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$")
+_SAFE_OLLAMA_FILE_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+
+def _is_generated_ollama_docker_exec_cmd(cmd: str | None) -> bool:
+    """Match only the fixed Docker exec shapes generated by Cookbook."""
+    if not cmd or any(char in cmd for char in _UNSAFE_DOCKER_EXEC_CHARS):
+        return False
+    try:
+        parts = shlex.split(cmd)
+    except ValueError:
+        return False
+    if len(parts) < 4 or parts[:2] != ["docker", "exec"]:
+        return False
+    container, executable = parts[2:4]
+    if container not in _OLLAMA_SIDECAR_CONTAINERS:
+        return False
+    if container == "ollama-rocm" and executable == "ollama":
+        return (
+            len(parts) == 6
+            and parts[4] == "show"
+            and _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(parts[5]) is not None
+        )
+    if container != "ollama-test" or executable != "ollama-import":
+        return False
+    if len(parts) not in {7, 8}:
+        return False
+    model, name, context_size = parts[4:7]
+    return (
+        _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(model) is not None
+        and _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(name) is not None
+        and re.fullmatch(r"[0-9]+", context_size) is not None
+        and (
+            len(parts) == 7
+            or _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(parts[7]) is not None
+        )
+    )
+
+
+def _missing_binary_message(
+    binary: str,
+    target: str,
+    *,
+    local_host_docker_blocked: bool = False,
+) -> str:
+    if binary == "tmux":
+        return (
+            f"tmux is required for Cookbook background downloads/serves on {target}. "
+            "Install it with your OS package manager, or run Cookbook server setup for that server."
+        )
+    if binary == "docker":
+        if local_host_docker_blocked:
+            return HOST_DOCKER_ACCESS_HINT
+        return (
+            f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. "
+            "Install Docker and make sure this user can run `docker`, then retry."
+        )
+    return f"{binary} is required on {target}, but it was not found."
+
+
+async def _remote_binary_available(
+    remote: str,
+    ssh_port: str | None,
+    binary: str,
+    *,
+    windows: bool = False,
+) -> bool:
+    port = ssh_port or ""
+    port_args = ["-p", port] if port and port != "22" else []
+    if windows:
+        check = f'powershell -NoProfile -Command "if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}"'
+    else:
+        check = f"command -v {shlex.quote(binary)} >/dev/null 2>&1"
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            "ssh",
+            "-o",
+            "ConnectTimeout=6",
+            "-o",
+            "StrictHostKeyChecking=no",
+            *port_args,
+            remote,
+            check,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await asyncio.wait_for(proc.communicate(), timeout=10)
+        return proc.returncode == 0
+    except Exception:
+        return False
+
+
+async def _binary_available(
+    binary: str,
+    remote: str | None,
+    ssh_port: str | None,
+    *,
+    windows: bool = False,
+    in_container: bool | None = None,
+    environ=None,
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+) -> bool:
+    if remote:
+        return await _remote_binary_available(
+            remote,
+            ssh_port,
+            binary,
+            windows=windows,
+        )
+    cli_available = shutil.which(binary) is not None
+    if binary != "docker":
+        return cli_available
+    return local_docker_available(
+        cli_available=cli_available,
+        in_container=in_container,
+        environ=environ,
+        socket_path=socket_path,
+    )
+
+
+
+def _local_ollama_docker_fallback_available(
+    *,
+    in_container: bool | None = None,
+    environ: dict[str, str] | None = None,
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+) -> bool:
+    return local_docker_available(
+        cli_available=shutil.which("docker") is not None,
+        in_container=in_container,
+        environ=environ,
+        socket_path=socket_path,
+    )
+
+
+def _local_ollama_docker_access_blocked(
+    *,
+    in_container: bool | None = None,
+    environ: dict[str, str] | None = None,
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+) -> bool:
+    containerized = running_in_container() if in_container is None else in_container
+    if not containerized or shutil.which("docker") is None:
+        return False
+    return not _local_ollama_docker_fallback_available(
+        in_container=containerized,
+        environ=environ,
+        socket_path=socket_path,
+    )
+
+
+def _append_local_ollama_download_command_lines(
+    lines: list[str],
+    ollama_cmd: str,
+    *,
+    docker_fallback_available: bool,
+    docker_fallback_blocked: bool,
+) -> None:
+    lines.append('if command -v ollama >/dev/null 2>&1; then')
+    lines.append(f'  ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}')
+    if docker_fallback_available:
+        lines.append('elif command -v docker >/dev/null 2>&1; then')
+        lines.append("  ODYSSEUS_OLLAMA_CONTAINER=\"$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(ollama-rocm|ollama-test)$' | head -1)\"")
+        lines.append('  if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then')
+        lines.append(f'    ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}')
+        lines.append('  fi')
+    elif docker_fallback_blocked:
+        hint = shlex.quote("ERROR: " + HOST_DOCKER_ACCESS_HINT)
+        lines.append('else')
+        lines.append(f"  printf '%s\\n' {hint}; exit 127")
+    lines.append('fi')
+    lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')
+
+
 def setup_cookbook_routes() -> APIRouter:
     router = APIRouter(tags=["cookbook"])
     _cookbook_state_path = Path(COOKBOOK_STATE_FILE)
+    _state_get_cache = {"ts": 0.0, "mtime": 0.0, "value": None}
+    _tasks_status_cache = {"ts": 0.0, "value": None}
+    _tasks_status_inflight = {"task": None}
 
     def _mask_secret(value: str) -> str:
         if not value:
@@ -408,46 +596,38 @@ def setup_cookbook_routes() -> APIRouter:
         safe_chmod(key_path.with_suffix(".pub"), 0o644)
         return {"ok": True, "public_key": _read_cookbook_public_key()}
 
+    class CookbookSshTestRequest(BaseModel):
+        host: str
+        ssh_port: str | None = None
+
+    @router.post("/api/cookbook/test-ssh")
+    async def test_cookbook_ssh(request: Request, req: CookbookSshTestRequest):
+        """Test a configured Cookbook SSH target without using generic shell exec."""
+        require_admin(request)
+        host = validate_remote_host(req.host)
+        ssh_port = validate_ssh_port(req.ssh_port)
+        try:
+            code, stdout, stderr = await run_ssh_command_async(
+                host,
+                ssh_port,
+                "echo ok",
+                timeout=8,
+                connect_timeout=5,
+                strict_host_key_checking=False,
+            )
+        except asyncio.TimeoutError:
+            return {"stdout": "", "stderr": "SSH test timed out", "exit_code": 124}
+        except Exception as e:
+            return {"stdout": "", "stderr": str(e), "exit_code": -1}
+        return {
+            "stdout": stdout.decode("utf-8", errors="replace"),
+            "stderr": stderr.decode("utf-8", errors="replace"),
+            "exit_code": code,
+        }
+
     def _needs_binary(cmd: str, binary: str) -> bool:
         return bool(re.search(rf"(^|[\s;&|()]){re.escape(binary)}($|[\s;&|()])", cmd or ""))
 
-    def _missing_binary_message(binary: str, target: str) -> str:
-        if binary == "tmux":
-            return (
-                f"tmux is required for Cookbook background downloads/serves on {target}. "
-                "Install it with your OS package manager, or run Cookbook server setup for that server."
-            )
-        if binary == "docker":
-            return (
-                f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. "
-                "Install Docker and make sure this user can run `docker`, then retry."
-            )
-        return f"{binary} is required on {target}, but it was not found."
-
-    async def _remote_binary_available(remote: str, ssh_port: str | None, binary: str, *, windows: bool = False) -> bool:
-        _port = ssh_port or ""
-        _pf = ["-p", _port] if _port and _port != "22" else []
-        if windows:
-            check = f"powershell -NoProfile -Command \"if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}\""
-        else:
-            check = f"command -v {shlex.quote(binary)} >/dev/null 2>&1"
-        try:
-            proc = await asyncio.create_subprocess_exec(
-                "ssh", "-o", "ConnectTimeout=6", "-o", "StrictHostKeyChecking=no",
-                *_pf, remote, check,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            )
-            await asyncio.wait_for(proc.communicate(), timeout=10)
-            return proc.returncode == 0
-        except Exception:
-            return False
-
-    async def _binary_available(binary: str, remote: str | None, ssh_port: str | None, *, windows: bool = False) -> bool:
-        if remote:
-            return await _remote_binary_available(remote, ssh_port, binary, windows=windows)
-        return shutil.which(binary) is not None
-
     def _launch_local_detached(session_id: str, bash_lines: list[str]) -> dict:
         """Windows-native stand-in for a LOCAL tmux session (tmux doesn't exist
         on Windows). Mirrors shell_routes._generate_win_detached / bg_jobs.launch:
@@ -576,15 +756,12 @@ def setup_cookbook_routes() -> APIRouter:
         # slower-but-reliable downloader (resumes cleanly from the .incomplete files).
         # Use `python3 -m pip` not `pip` — macOS has no bare `pip` command.
         if is_ollama_download:
-            lines.append('if command -v ollama >/dev/null 2>&1; then')
-            lines.append(f'  ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}')
-            lines.append('elif command -v docker >/dev/null 2>&1; then')
-            lines.append('  ODYSSEUS_OLLAMA_CONTAINER="$(docker ps --format \'{{.Names}}\' 2>/dev/null | grep -E \'^(ollama-rocm|ollama-test)$\' | head -1)"')
-            lines.append('  if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then')
-            lines.append(f'    ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}')
-            lines.append('  fi')
-            lines.append('fi')
-            lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')
+            _append_local_ollama_download_command_lines(
+                lines,
+                ollama_cmd,
+                docker_fallback_available=_local_ollama_docker_fallback_available(),
+                docker_fallback_blocked=_local_ollama_docker_access_blocked(),
+            )
         else:
             lines.append(f"command -v hf >/dev/null 2>&1 || {_pip_install_fallback_chain('huggingface_hub', upgrade=True)}")
             if req.disable_hf_transfer:
@@ -902,10 +1079,16 @@ def setup_cookbook_routes() -> APIRouter:
                 cwd=str(Path.home()),
             )
         stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=60)
+        stderr_txt = stderr_b.decode(errors="replace").strip()
+        stdout_txt = stdout_b.decode(errors="replace").strip()
+        if proc.returncode != 0:
+            msg = stderr_txt or f"Cached model scan failed with exit code {proc.returncode}"
+            logger.warning(f"Cached model scan failed host={host or 'local'} rc={proc.returncode}: {msg[:500]}")
+            return {"models": [], "host": host or "local", "error": msg}
 
         models = []
         try:
-            raw = json.loads(stdout_b.decode(errors="replace").strip())
+            raw = json.loads(stdout_txt)
             for m in raw:
                 size_gb = m["size_bytes"] / (1024 ** 3)
                 if size_gb >= 1:
@@ -933,8 +1116,11 @@ def setup_cookbook_routes() -> APIRouter:
                     entry["gguf_files"] = m["gguf_files"]
                 models.append(entry)
         except Exception as e:
-            logger.warning(f"Failed to parse cached models: {e}")
-            logger.warning(f"stderr: {stderr_b.decode(errors='replace')[:500]}")
+            logger.warning(f"Failed to parse cached models host={host or 'local'}: {e}")
+            if stderr_txt:
+                logger.warning(f"stderr: {stderr_txt[:500]}")
+            msg = stderr_txt or stdout_txt[:500] or str(e)
+            return {"models": [], "host": host or "local", "error": msg}
 
         return {"models": models, "host": host or "local"}
 
@@ -1127,6 +1313,22 @@ def setup_cookbook_routes() -> APIRouter:
                 try:
                     ep = db.query(_ME).filter(_ME.id == endpoint_id).first()
                     if ep:
+                        # A scheduled serve can leave old non-zero exit markers
+                        # in tmux scrollback while the current OpenAI endpoint is
+                        # actually alive. Verify reachability before deleting the
+                        # endpoint row; otherwise chats fall back even though the
+                        # served model is ready.
+                        try:
+                            probe_url = ep.base_url.rstrip("/") + "/models"
+                            with urllib.request.urlopen(probe_url, timeout=3) as resp:
+                                if 200 <= getattr(resp, "status", 0) < 300:
+                                    logger.info(
+                                        f"crash-watchdog: serve {session_id} has exit marker {exit_code} "
+                                        f"but endpoint {ep.id} is reachable; leaving it registered"
+                                    )
+                                    return
+                        except Exception:
+                            pass
                         logger.info(
                             f"crash-watchdog: dropping endpoint {endpoint_id} "
                             f"({ep.name} @ {ep.base_url}) — serve exited {exit_code}"
@@ -1213,6 +1415,8 @@ def setup_cookbook_routes() -> APIRouter:
                 existing.is_enabled = True
                 existing.model_type = "llm"
                 existing.name = display_name
+                existing.endpoint_kind = "local"
+                existing.model_refresh_mode = "auto"
                 if is_ollama_endpoint:
                     existing.endpoint_kind = "ollama"
                     if pinned_models:
@@ -1260,7 +1464,8 @@ def setup_cookbook_routes() -> APIRouter:
                 api_key=None,
                 is_enabled=True,
                 model_type="llm",
-                endpoint_kind="ollama" if is_ollama_endpoint else "auto",
+                endpoint_kind="ollama" if is_ollama_endpoint else "local",
+                model_refresh_mode="auto",
                 cached_models=json.dumps(pinned_models) if pinned_models else None,
                 pinned_models=json.dumps(pinned_models) if pinned_models else None,
                 supports_tools=supports_tools,
@@ -1322,13 +1527,18 @@ def setup_cookbook_routes() -> APIRouter:
         req.gpus = _validate_gpus(req.gpus)
         req.hf_token = req.hf_token or _load_stored_hf_token()
         _validate_token(req.hf_token)
-        # Normalize away backslash-newline continuations (multi-line pasted
-        # serve commands) so the cleaned single-line command is what gets
-        # written into the runner script and used for engine auto-detection.
-        # `_validate_serve_cmd` returns None for empty input; coerce to "" so the
-        # many downstream `"engine" in req.cmd` membership checks can't hit
-        # `TypeError: argument of type 'NoneType'` (a 500 instead of a clean 400).
-        req.cmd = _validate_serve_cmd(req.cmd) or ""
+        # Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
+        # Keep Docker out of the general allowlist: only these parsed shapes may
+        # proceed to the target-aware Docker availability/opt-in preflight.
+        if _is_generated_ollama_docker_exec_cmd(req.cmd):
+            req.cmd = req.cmd.strip()
+        else:
+            # Normalize away backslash-newline continuations (multi-line pasted
+            # serve commands) so the cleaned single-line command is what gets
+            # written into the runner script and used for engine auto-detection.
+            # `_validate_serve_cmd` returns None for empty input; coerce to "" so
+            # downstream `"engine" in req.cmd` checks cannot raise TypeError.
+            req.cmd = _validate_serve_cmd(req.cmd) or ""
         req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or ""
         req.cmd = _normalize_minimax_m3_vllm_cmd(req.cmd)
         req.cmd = _venv_safe_local_pip_install_cmd(
@@ -1406,9 +1616,18 @@ def setup_cookbook_routes() -> APIRouter:
                 "session_id": session_id,
             }
         if _needs_binary(req.cmd, "docker") and not await _binary_available("docker", remote, req.ssh_port, windows=is_windows):
+            local_host_docker_blocked = (
+                not remote
+                and running_in_container()
+                and not host_docker_access_enabled()
+            )
             return {
                 "ok": False,
-                "error": _missing_binary_message("docker", remote or "local server"),
+                "error": _missing_binary_message(
+                    "docker",
+                    remote or "local server",
+                    local_host_docker_blocked=local_host_docker_blocked,
+                ),
                 "session_id": session_id,
             }
 
@@ -1642,7 +1861,10 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('  exec 3<&-; exec 3>&-')
                 runner_lines.append('done')
                 runner_lines.append('if ! command -v ollama &>/dev/null; then')
-                runner_lines.append('  echo "ERROR: Ollama not found on this server. Install it from https://ollama.com/download or `curl -fsSL https://ollama.com/install.sh | sh`."')
+                # Single-quoted on purpose: backticks inside a double-quoted
+                # echo are command substitution, and this line used to run the
+                # curl|sh installer on the target host instead of printing it.
+                runner_lines.append(f"  echo '{_bash_squote(OLLAMA_MISSING_HINT)}'")
                 runner_lines.append('  echo')
                 runner_lines.append('  echo "=== Process exited with code 127 ==="')
                 runner_lines.append('  exec bash -i')
@@ -2405,12 +2627,29 @@ def setup_cookbook_routes() -> APIRouter:
     async def get_cookbook_state(request: Request):
         """Load saved cookbook state (tasks, servers, presets, settings)."""
         require_admin(request)
+        now = time.monotonic()
+        try:
+            mtime = _cookbook_state_path.stat().st_mtime if _cookbook_state_path.exists() else 0.0
+        except Exception:
+            mtime = 0.0
+        cached = _state_get_cache.get("value")
+        if cached is not None and _state_get_cache.get("mtime") == mtime and now - float(_state_get_cache.get("ts") or 0) < 1.5:
+            return cached
         if _cookbook_state_path.exists():
             try:
-                return _state_for_client(json.loads(_cookbook_state_path.read_text(encoding="utf-8")))
+                state = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
+                saved_tasks = state.get("tasks", [])
+                tasks = saved_tasks if isinstance(saved_tasks, list) else list(saved_tasks.values()) if isinstance(saved_tasks, dict) else []
+                client_state = _state_for_client(state)
+                _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state})
+                return client_state
             except Exception:
-                return _state_for_client({})
-        return _state_for_client({})
+                client_state = _state_for_client({})
+                _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state})
+                return client_state
+        client_state = _state_for_client({})
+        _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state})
+        return client_state
 
     @router.post("/api/cookbook/state")
     async def save_cookbook_state(request: Request):
@@ -2518,7 +2757,19 @@ def setup_cookbook_routes() -> APIRouter:
                             f"not in incoming body (race guard): "
                             f"{[t.get('sessionId') for t in preserved]}")
                 data["tasks"] = incoming_tasks + preserved
-            atomic_write_json(str(_cookbook_state_path), _state_for_storage(data, on_disk), indent=2)
+            storage_state = _state_for_storage(data, on_disk)
+            if storage_state == on_disk:
+                return {"ok": True, "preserved": len(preserved), "unchanged": True}
+            atomic_write_json(str(_cookbook_state_path), storage_state, indent=2)
+            try:
+                mtime = _cookbook_state_path.stat().st_mtime
+                _state_get_cache.update({
+                    "ts": time.monotonic(),
+                    "mtime": mtime,
+                    "value": _state_for_client(storage_state),
+                })
+            except Exception:
+                pass
             return {"ok": True, "preserved": len(preserved)}
         except Exception as e:
             return {"ok": False, "error": str(e)}
@@ -2640,10 +2891,10 @@ def setup_cookbook_routes() -> APIRouter:
 
         return {"models": out}
 
-    # Rate-limit for the orphan-tmux adoption sweep. 60s interval so SSH
+    # Rate-limit for the orphan-tmux adoption sweep. Five-minute interval so SSH
     # work is genuinely sparse even on an actively-polled cookbook page.
     _last_orphan_sweep_ts = [0.0]
-    _ORPHAN_SWEEP_MIN_INTERVAL_S = 60.0
+    _ORPHAN_SWEEP_MIN_INTERVAL_S = 300.0
     # Concurrency guard so two requests racing don't both spawn a sweep.
     _orphan_sweep_inflight = [False]
 
@@ -2737,6 +2988,54 @@ def setup_cookbook_routes() -> APIRouter:
                     continue
                 if sid in known_sids:
                     continue
+                try:
+                    cap = subprocess.run(
+                        ssh_base + [host, "tmux", "capture-pane", "-t", sid, "-p", "-S", "-300"],
+                        timeout=6, capture_output=True, text=True,
+                    )
+                    pane = cap.stdout or ""
+                except Exception:
+                    pane = ""
+
+                if sid.startswith("cookbook-"):
+                    repo_id = ""
+                    try:
+                        script = subprocess.run(
+                            ssh_base + [host, "cat", f".{sid}_run.sh"],
+                            timeout=6, capture_output=True, text=True,
+                        )
+                        script_text = script.stdout or ""
+                    except Exception:
+                        script_text = ""
+                    m_repo = re.search(r"repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text)
+                    if not m_repo:
+                        m_repo = re.search(r"snapshot_download\(\s*repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text)
+                    if not m_repo:
+                        m_repo = re.search(r"(?:https://huggingface\.co/)?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", script_text)
+                    repo_id = m_repo.group(1) if m_repo else f"adopted:{sid}"
+                    import time as _t2
+                    tasks.append({
+                        "id": sid,
+                        "sessionId": sid,
+                        "name": repo_id.split("/")[-1] if "/" in repo_id else repo_id,
+                        "type": "download",
+                        "status": "running",
+                        "output": (pane or f"Auto-adopted from orphan tmux download session on {host}.")[-5000:],
+                        "ts": int(_t2.time() * 1000),
+                        "payload": {
+                            "repo_id": repo_id,
+                            "remote_host": host,
+                            "_cmd": "(orphan tmux download - original launch cmd recovered from tmux/session only)",
+                        },
+                        "remoteHost": host,
+                        "sshPort": sport,
+                        "platform": "linux",
+                        "_adoptedExternally": True,
+                    })
+                    known_sids.add(sid)
+                    adopted_any = True
+                    logger.info(f"auto-adopted orphan download tmux session {sid!r} on {host}")
+                    continue
                 # Adopt any session whose pane is currently running a
                 # known model-server process (checked below). The earlier
                 # prefix gate (serve-/cookbook-) dropped legitimate
@@ -2766,14 +3065,6 @@ def setup_cookbook_routes() -> APIRouter:
                 # Try to recover a plausible repo_id + port from the
                 # pane buffer. Cheap heuristic — if we can't, register
                 # with placeholder fields; the UI still shows it.
-                try:
-                    cap = subprocess.run(
-                        ssh_base + [host, "tmux", "capture-pane", "-t", sid, "-p", "-S", "-300"],
-                        timeout=6, capture_output=True, text=True,
-                    )
-                    pane = cap.stdout or ""
-                except Exception:
-                    pane = ""
                 import re as _re_orphan
                 # vLLM banner: "model   /path/...". Falls back to the
                 # raw vllm-serve command if the banner already scrolled.
@@ -3158,11 +3449,52 @@ def setup_cookbook_routes() -> APIRouter:
         event loop. Now the whole body runs in a worker thread via
         asyncio.to_thread so other requests stay responsive."""
         require_admin(request)
-        return await asyncio.to_thread(_cookbook_tasks_status_sync)
+        now = time.monotonic()
+        cached = _tasks_status_cache.get("value")
+        if cached is not None and now - float(_tasks_status_cache.get("ts") or 0) < 2.0:
+            return cached
+        inflight = _tasks_status_inflight.get("task")
+        if inflight and not inflight.done():
+            return await inflight
+
+        async def _compute():
+            data = await asyncio.to_thread(_cookbook_tasks_status_sync)
+            _tasks_status_cache.update({"ts": time.monotonic(), "value": data})
+            return data
+
+        task = asyncio.create_task(_compute())
+        _tasks_status_inflight["task"] = task
+        try:
+            return await task
+        finally:
+            if _tasks_status_inflight.get("task") is task:
+                _tasks_status_inflight["task"] = None
 
     def _cookbook_tasks_status_sync():
         import subprocess
 
+        def _pick_download_progress(lines: list[str]) -> str:
+            """Pick the most useful live HF progress line from a tmux pane."""
+            if not lines:
+                return ""
+            downloading_lines = [l for l in lines if l.startswith("Downloading")]
+            if downloading_lines:
+                return downloading_lines[-1]
+            progress_lines = [
+                l for l in lines
+                if re.search(r"\b(?:100|[1-9]?\d)%", l)
+                and (
+                    "<" in l
+                    or "it/s" in l
+                    or "B/s" in l
+                    or "safetensors" in l
+                    or ".gguf" in l.lower()
+                )
+            ]
+            if progress_lines:
+                return progress_lines[-1]
+            return lines[-1]
+
         def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool:
             """Best-effort check for a completed HF cache entry.
 
@@ -3344,11 +3676,7 @@ def setup_cookbook_routes() -> APIRouter:
                             encoding="utf-8", errors="replace"
                         ).strip()[-12000:]
                         lines = [l.strip() for l in full_snapshot.split('\n') if l.strip()]
-                        downloading_lines = [l for l in lines if l.startswith("Downloading")]
-                        if downloading_lines:
-                            progress_text = downloading_lines[-1]
-                        elif lines:
-                            progress_text = lines[-1]
+                        progress_text = _pick_download_progress(lines)
                 except Exception:
                     pass
             else:
@@ -3382,11 +3710,7 @@ def setup_cookbook_routes() -> APIRouter:
                             if cap.returncode == 0:
                                 full_snapshot = cap.stdout.strip()
                                 lines = [l.strip() for l in full_snapshot.split('\n') if l.strip()]
-                                downloading_lines = [l for l in lines if l.startswith("Downloading")]
-                                if downloading_lines:
-                                    progress_text = downloading_lines[-1]
-                                elif lines:
-                                    progress_text = lines[-1]
+                                progress_text = _pick_download_progress(lines)
                         except Exception:
                             pass
 

routes/document_helpers.py

@@ -29,6 +29,7 @@ class DocumentCreate(BaseModel):
 class DocumentUpdate(BaseModel):
     content: str
     summary: Optional[str] = None
+    force_version: bool = False
 
 class DocumentPatch(BaseModel):
     title: Optional[str] = None

routes/document_routes.py

@@ -570,8 +570,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 raise HTTPException(404, "Document not found")
             _verify_doc_owner(db, doc, user)
 
-            # Skip if content is identical
-            if doc.current_content == req.content:
+            # Skip if content is identical unless the caller explicitly wants
+            # a checkpoint version from the current editor state.
+            if doc.current_content == req.content and not req.force_version:
                 return _doc_to_dict(doc)
 
             _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
@@ -583,7 +584,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
 
             now = datetime.now(timezone.utc)
             coalesced = False
-            if latest_ver and latest_ver.source == "user":
+            if latest_ver and latest_ver.source == "user" and not req.force_version:
                 ver_time = latest_ver.created_at
                 if ver_time.tzinfo is None:
                     ver_time = ver_time.replace(tzinfo=timezone.utc)
@@ -799,10 +800,26 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
             from src.document_actions import _JUNK_TITLES
 
             to_delete = []
+            now = datetime.now(timezone.utc)
             for doc in docs:
+                created = doc.created_at
+                if created and created.tzinfo is None:
+                    created = created.replace(tzinfo=timezone.utc)
+
+                # Skip freshly created documents to avoid deleting them while the user is actively editing
+                if created and (now - created).total_seconds() < 900:  # 15 minutes
+                    continue
+
                 content = (doc.current_content or "").strip()
                 title_raw = (doc.title or "").strip()
                 title = title_raw.lower()
+                is_fresh_empty = (
+                    not content
+                    and created is not None
+                    and (now - created).total_seconds() < 1800
+                )
+                if is_fresh_empty:
+                    continue
 
                 # Strip markdown noise to get a "real" character count
                 stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE)
@@ -837,10 +854,6 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                     to_delete.append(doc); deleted += 1; continue
                 if title in _JUNK_TITLES:
                     to_delete.append(doc); deleted += 1; continue
-                if real_len < 30:
-                    to_delete.append(doc); deleted += 1; continue
-                if "\n" not in content and real_len < 50:
-                    to_delete.append(doc); deleted += 1; continue
 
                 # Fix empty or placeholder titles on survivors
                 if not title_raw or title_raw == "Untitled":

routes/email_helpers.py

@@ -424,12 +424,19 @@ SCHEDULED_DB = Path(SCHEDULED_EMAILS_DB)
 OWNER_SCOPED_EMAIL_CACHE_TABLES = {
     "email_summaries",
     "email_ai_replies",
+    "email_translations",
     "email_calendar_extractions",
     "email_urgency_alerts",
     "sender_signatures",
 }
 
 
+def email_translation_body_hash(body: str) -> str:
+    import hashlib as _hashlib
+    normalized = (body or "").strip()
+    return _hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest()
+
+
 def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
     owner = (owner or "").strip()
     if owner:
@@ -437,14 +444,34 @@ def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
     return "(owner = '' OR owner IS NULL)", ()
 
 
-def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, columns: list[str]):
+def _ensure_owner_scoped_email_cache_table(
+    conn,
+    table: str,
+    create_sql: str,
+    columns: list[str],
+    pk_columns: list[str] | None = None,
+):
     """Rebuild legacy Message-ID-only cache tables with owner in the PK."""
+    desired_pk_cols = pk_columns or ["message_id", "owner"]
     conn.execute(create_sql)
     try:
         info = conn.execute(f"PRAGMA table_info({table})").fetchall()
         cols = [r[1] for r in info]
         pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
-        if "owner" in cols and pk_cols == ["message_id", "owner"]:
+        for col in columns:
+            if col not in cols:
+                if col == "owner":
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN owner TEXT DEFAULT ''")
+                elif col in {"event_uids"}:
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT '[]'")
+                elif col.startswith("has_") or col.endswith("_created") or col.endswith("_count"):
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} INTEGER DEFAULT 0")
+                elif col == "created_at":
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT ''")
+                else:
+                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT")
+                cols.append(col)
+        if "owner" in cols and pk_cols == desired_pk_cols:
             return
 
         conn.execute(f"ALTER TABLE {table} RENAME TO {table}__old")
@@ -577,6 +604,25 @@ def _init_scheduled_db():
             PRIMARY KEY (message_id, owner)
         )
     """, ["message_id", "owner", "uid", "folder", "reply", "model_used", "created_at"])
+    _ensure_owner_scoped_email_cache_table(conn, "email_translations", """
+        CREATE TABLE IF NOT EXISTS email_translations (
+            body_hash TEXT,
+            owner TEXT DEFAULT '',
+            target_language TEXT DEFAULT 'English',
+            uid TEXT,
+            folder TEXT,
+            subject TEXT,
+            sender TEXT,
+            translation TEXT,
+            same_language INTEGER DEFAULT 0,
+            model_used TEXT,
+            created_at TEXT NOT NULL,
+            PRIMARY KEY (body_hash, owner, target_language)
+        )
+    """, [
+        "body_hash", "owner", "target_language", "uid", "folder", "subject", "sender",
+        "translation", "same_language", "model_used", "created_at",
+    ], ["body_hash", "owner", "target_language"])
     # Email tags / spam classification cache. SECURITY: keyed by
     # (message_id, owner) because Message-IDs are GLOBAL (a newsletter goes
     # to many users with the same Message-ID). Without owner-scoping, a
@@ -586,6 +632,7 @@ def _init_scheduled_db():
         CREATE TABLE IF NOT EXISTS email_tags (
             message_id TEXT,
             owner TEXT DEFAULT '',
+            account_id TEXT DEFAULT '',
             uid TEXT,
             folder TEXT,
             subject TEXT,
@@ -596,7 +643,7 @@ def _init_scheduled_db():
             moved_to TEXT,
             model_used TEXT,
             created_at TEXT NOT NULL,
-            PRIMARY KEY (message_id, owner)
+            PRIMARY KEY (message_id, owner, account_id)
         )
     """)
     # Backfill migration: older installs created the table with
@@ -604,28 +651,35 @@ def _init_scheduled_db():
     # promote it into the PK by rebuild-copy-swap (SQLite can't ALTER PK).
     try:
         _cols = [r[1] for r in conn.execute("PRAGMA table_info(email_tags)")]
+        _pk_cols = [r[1] for r in sorted(conn.execute("PRAGMA table_info(email_tags)").fetchall(), key=lambda row: row[5] or 99) if r[5]]
         if "owner" not in _cols:
-            # Add the column first so reads/writes don't break mid-migration.
             conn.execute("ALTER TABLE email_tags ADD COLUMN owner TEXT DEFAULT ''")
-            # Rebuild with composite PK. Existing rows get owner='' (legacy
-            # single-user); the urgency scanner will overwrite as it
-            # re-classifies. No data loss.
+            _cols.append("owner")
+        if "account_id" not in _cols:
+            conn.execute("ALTER TABLE email_tags ADD COLUMN account_id TEXT DEFAULT ''")
+            _cols.append("account_id")
+        if _pk_cols != ["message_id", "owner", "account_id"]:
+            # Rebuild with account-aware composite PK. Existing rows get
+            # account_id='' and are still readable as legacy fallback rows;
+            # fresh task runs write exact account ids and no longer block each
+            # other when two accounts share a Message-ID.
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS email_tags__new (
                     message_id TEXT,
                     owner TEXT DEFAULT '',
+                    account_id TEXT DEFAULT '',
                     uid TEXT, folder TEXT, subject TEXT, sender TEXT,
                     tags TEXT, spam_verdict INTEGER DEFAULT 0,
                     spam_reason TEXT, moved_to TEXT, model_used TEXT,
                     created_at TEXT NOT NULL,
-                    PRIMARY KEY (message_id, owner)
+                    PRIMARY KEY (message_id, owner, account_id)
                 )
             """)
             conn.execute("""
                 INSERT OR IGNORE INTO email_tags__new
-                  (message_id, owner, uid, folder, subject, sender, tags,
+                  (message_id, owner, account_id, uid, folder, subject, sender, tags,
                    spam_verdict, spam_reason, moved_to, model_used, created_at)
-                SELECT message_id, COALESCE(owner, ''), uid, folder, subject,
+                SELECT message_id, COALESCE(owner, ''), COALESCE(account_id, ''), uid, folder, subject,
                        sender, tags, spam_verdict, spam_reason, moved_to,
                        model_used, created_at
                 FROM email_tags
@@ -641,11 +695,12 @@ def _init_scheduled_db():
             message_id TEXT,
             owner TEXT DEFAULT '',
             uid TEXT,
+            event_uids TEXT DEFAULT '[]',
             events_created INTEGER DEFAULT 0,
             created_at TEXT NOT NULL,
             PRIMARY KEY (message_id, owner)
         )
-    """, ["message_id", "owner", "uid", "events_created", "created_at"])
+    """, ["message_id", "owner", "uid", "event_uids", "events_created", "created_at"])
     _ensure_owner_scoped_email_cache_table(conn, "email_urgency_alerts", """
         CREATE TABLE IF NOT EXISTS email_urgency_alerts (
             message_id TEXT,
@@ -671,6 +726,64 @@ def _init_scheduled_db():
             PRIMARY KEY (owner, account_key, folder, message_key)
         )
     """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS email_message_index (
+            owner TEXT NOT NULL DEFAULT '',
+            account_key TEXT NOT NULL DEFAULT '',
+            folder TEXT NOT NULL,
+            uid TEXT NOT NULL,
+            message_id TEXT,
+            subject TEXT,
+            from_name TEXT,
+            from_address TEXT,
+            to_text TEXT,
+            cc_text TEXT,
+            date_iso TEXT,
+            date_display TEXT,
+            date_epoch REAL DEFAULT 0,
+            size INTEGER DEFAULT 0,
+            flags TEXT DEFAULT '',
+            has_attachments INTEGER DEFAULT 0,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (owner, account_key, folder, uid)
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
+        ON email_message_index(owner, account_key, folder, date_epoch DESC)
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS ix_email_message_index_message_id
+        ON email_message_index(owner, account_key, message_id)
+    """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS email_body_preview_cache (
+            owner TEXT NOT NULL DEFAULT '',
+            account_key TEXT NOT NULL DEFAULT '',
+            folder TEXT NOT NULL,
+            uid TEXT NOT NULL,
+            message_id TEXT,
+            payload_json TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (owner, account_key, folder, uid)
+        )
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS ix_email_body_preview_message_id
+        ON email_body_preview_cache(owner, account_key, message_id)
+    """)
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS email_attachment_metadata_cache (
+            owner TEXT NOT NULL DEFAULT '',
+            account_key TEXT NOT NULL DEFAULT '',
+            folder TEXT NOT NULL,
+            uid TEXT NOT NULL,
+            message_id TEXT,
+            attachments_json TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (owner, account_key, folder, uid)
+        )
+    """)
     # Boundary cache — LLM-detected sig/quote start positions in the body.
     # Stored as char offsets (-1 = no boundary found). Once cached, the
     # client uses these to fold without ever re-calling the LLM.
@@ -1160,10 +1273,15 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str
     try:
         c = _imap_connect(account_id, owner=owner)
         c.select(_q(src))
-        status, _ = c.copy(uid, _q(dest))
+        # Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy()
+        # and store() operate on message SEQUENCE NUMBERS, so addressing them
+        # with a UID moved/deleted the wrong message (or silently no-oped when
+        # the UID exceeded the message count). Use the UID commands, matching
+        # the move/delete path in email_routes.py.
+        status, _ = c.uid("COPY", uid, _q(dest))
         if status != "OK":
             return False
-        c.store(uid, "+FLAGS", "\\Deleted")
+        c.uid("STORE", uid, "+FLAGS", "\\Deleted")
         c.expunge()
         return True
     except Exception as e:
@@ -1276,12 +1394,14 @@ def _list_attachments_from_msg(msg):
             except Exception:
                 payload = b""
         size = len(payload) if payload is not None else 0
+        content_id = (part.get("Content-ID") or "").strip().strip("<>")
         attachments.append({
             "index": idx,
             "filename": filename,
             "content_type": ct,
             "size": size,
             "is_inline": "inline" in cd.lower(),
+            "content_id": content_id,
         })
         idx += 1
     return attachments
@@ -1720,6 +1840,10 @@ class SendEmailRequest(BaseModel):
     attachments: Optional[List[str]] = None
     # Which account to send from. None = default account.
     account_id: Optional[str] = None
+    # Source message for replies. When present, /send marks this exact message
+    # answered after successful delivery so it leaves undone/reply-soon views.
+    source_uid: Optional[str] = None
+    source_folder: Optional[str] = None
     # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
     odysseus_kind: Optional[str] = None
     # If true, /send waits for SMTP + Sent append and returns the sent UID.

routes/email_pollers.py

@@ -29,7 +29,7 @@ from datetime import datetime
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
-from src.llm_core import llm_call_async
+from src.task_endpoint import resolve_task_candidates, task_llm_call_async
 
 from routes.email_helpers import (
     _strip_think, _extract_reply, _apply_email_style_mechanics, _load_settings, _save_settings, _get_email_config,
@@ -56,6 +56,35 @@ _CAL_ACTION_ARRAY_RE = re.compile(
 )
 
 
+def _extract_json_array_from_text(text: str):
+    """Return the last valid JSON array embedded in model output, if any."""
+    if not text:
+        return None
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip(), flags=re.MULTILINE).strip()
+    decoder = json.JSONDecoder()
+    try:
+        parsed = decoder.decode(cleaned)
+        if isinstance(parsed, list):
+            return parsed
+    except Exception:
+        pass
+
+    # Models often explain themselves and finish with `[]` or `[{"action":...}]`.
+    # Scan every array opener and keep the last complete JSON array, rather than
+    # using a greedy regex that can swallow prose containing square brackets.
+    last = None
+    for idx, ch in enumerate(cleaned):
+        if ch != "[":
+            continue
+        try:
+            parsed, _end = decoder.raw_decode(cleaned[idx:])
+        except Exception:
+            continue
+        if isinstance(parsed, list):
+            last = parsed
+    return last
+
+
 def _owner_for_email_account(account_id: str | None) -> str:
     if not account_id:
         return ""
@@ -88,6 +117,8 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
                                    do_tag: bool = False, do_spam: bool = False,
                                    do_calendar: bool = False,
                                    days_back: int = 1,
+                                   account_id: str | None = None,
+                                   max_process: int | None = None,
                                    progress_cb=None) -> str:
     """One iteration of the email scan. Temporarily flips settings flags
     so the existing background-loop logic runs exactly once for the requested ops."""
@@ -102,7 +133,12 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
     settings["email_auto_calendar"] = bool(do_calendar)
     _save_settings(settings)
     try:
-        return await _auto_summarize_pass(days_back=days_back, progress_cb=progress_cb)
+        return await _auto_summarize_pass(
+            days_back=days_back,
+            account_id=account_id,
+            max_process=max_process,
+            progress_cb=progress_cb,
+        )
     finally:
         s2 = _load_settings()
         for k, v in prev.items():
@@ -140,7 +176,7 @@ def _latest_inbox_fallback_uids(conn, reconnect):
         return [], reconnect()
 
 
-async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
     """Single pass of the auto-summarize/reply scan.
 
     When account_id is None, iterates over every enabled account in
@@ -167,28 +203,41 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
             names = {}
         if len(ids) <= 1:
             # Single-account (or zero rows — fallback to legacy settings.json lookup)
-            return await _auto_summarize_pass_single(days_back=days_back, account_id=(ids[0] if ids else None), progress_cb=progress_cb)
+            return await _auto_summarize_pass_single(
+                days_back=days_back,
+                account_id=(ids[0] if ids else None),
+                max_process=max_process,
+                progress_cb=progress_cb,
+            )
         outs = []
         for idx, aid in enumerate(ids, start=1):
             try:
                 await _emit_progress(progress_cb, f"{names.get(aid, aid[:8])}: starting ({idx}/{len(ids)})")
-                result = await _auto_summarize_pass_single(days_back=days_back, account_id=aid, progress_cb=progress_cb)
+                result = await _auto_summarize_pass_single(
+                    days_back=days_back,
+                    account_id=aid,
+                    max_process=max_process,
+                    progress_cb=progress_cb,
+                )
                 outs.append(f"[{names.get(aid, aid[:8])}] {result}")
             except Exception as e:
                 logger.warning(f"auto-summarize pass failed for account {aid}: {e}")
                 outs.append(f"[{names.get(aid, aid[:8])}] error: {e}")
         return "\n".join(outs)
-    return await _auto_summarize_pass_single(days_back=days_back, account_id=account_id, progress_cb=progress_cb)
+    return await _auto_summarize_pass_single(
+        days_back=days_back,
+        account_id=account_id,
+        max_process=max_process,
+        progress_cb=progress_cb,
+    )
 
 
-async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
     """Single pass of the auto-summarize/reply scan for ONE account.
     Reads current settings flags."""
     import asyncio
     import sqlite3 as _sql3
-    import requests as _req
-    from src.endpoint_resolver import resolve_endpoint
-    from src.llm_core import _uses_max_completion_tokens, _restricts_temperature
+    from src.llm_core import _uses_max_completion_tokens
 
     settings = _load_settings()
     auto_sum = settings.get("email_auto_summarize", False)
@@ -265,9 +314,15 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         ).fetchall()}
         if auto_tag or auto_spam:
             if account_owner:
-                _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner=?", (account_owner,)).fetchall()}
+                _tag_existing = {r[0] for r in _c.execute(
+                    "SELECT message_id FROM email_tags WHERE owner=? AND (account_id=? OR account_id='' OR account_id IS NULL)",
+                    (account_owner, account_id or ""),
+                ).fetchall()}
             else:
-                _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner='' OR owner IS NULL").fetchall()}
+                _tag_existing = {r[0] for r in _c.execute(
+                    "SELECT message_id FROM email_tags WHERE (owner='' OR owner IS NULL) AND (account_id=? OR account_id='' OR account_id IS NULL)",
+                    (account_id or "",),
+                ).fetchall()}
         else:
             _tag_existing = set()
         _cal_existing = {r[0] for r in _c.execute(
@@ -296,11 +351,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         if auto_spam and not spam_folder:
             logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move")
 
-        url, model, headers = resolve_endpoint("utility", owner=account_owner)
-        if not url:
-            url, model, headers = resolve_endpoint("default", owner=account_owner)
-        if not url or not model:
+        task_candidates = resolve_task_candidates(owner=account_owner)
+        if not task_candidates:
             return "No model configured"
+        url, model, headers = task_candidates[0]
 
         writing_style = settings.get("email_writing_style", "")
         processed = 0
@@ -314,7 +368,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         _reply_failed = 0
         _detail_lines = []
         _current_folder = "INBOX"
-        _max_process = 5
+        # Calendar extraction is sequential and each row can involve a model
+        # call plus a calendar write. Keep the scheduled calendar-only pass
+        # below the 5-minute action budget instead of timing out mid-run.
+        _default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5
+        try:
+            _max_process = max(1, int(max_process)) if max_process is not None else _default_max_process
+        except Exception:
+            _max_process = _default_max_process
         for _entry in uid_list:
             if processed >= _max_process:
                 break
@@ -406,48 +467,30 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     req_headers.update(headers)
 
                 if need_sum:
-                    tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
-                    payload = {
-                        "model": model,
-                        "messages": [
-                            {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
-                            {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
-                        ],
-                        tok_key: 16384,
-                        "temperature": 0.3,
-                        "stream": False,
-                    }
-                    # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
-                    if _restricts_temperature(model):
-                        payload.pop("temperature", None)
                     try:
-                        # Use to_thread so this sync HTTP call doesn't freeze
-                        # the entire event loop while the LLM thinks (240s).
-                        resp = await asyncio.to_thread(
-                            _req.post, url, json=payload, headers=req_headers, timeout=240
+                        summary = await task_llm_call_async(
+                            messages=[
+                                {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
+                                {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
+                            ],
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.3, max_tokens=16384, timeout=240,
                         )
-                        if resp.ok:
-                            rdata = resp.json()
-                            m = (rdata.get("choices") or [{}])[0].get("message", {})
-                            summary = (m.get("content") or "").strip()
-                            summary = _extract_reply(summary)
-                            if not summary:
-                                rc = (m.get("reasoning_content") or "").strip()
-                                bullets = [ln.strip() for ln in rc.split("\n") if re.match(r"^[-•*]\s+|^\d+[.)]\s+", ln.strip())]
-                                summary = "\n".join(bullets) if bullets else ""
-                            if summary:
-                                _c = _sql3.connect(SCHEDULED_DB)
-                                _c.execute("""
-                                    INSERT OR REPLACE INTO email_summaries
-                                    (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
-                                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-                                """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
-                                _c.commit()
-                                _c.close()
-                                _sum_existing.add(message_id)
-                                _summaries_created += 1
-                                _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
-                                _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
+                        summary = _extract_reply((summary or "").strip())
+                        if summary:
+                            _c = _sql3.connect(SCHEDULED_DB)
+                            _c.execute("""
+                                INSERT OR REPLACE INTO email_summaries
+                                (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
+                                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
+                            _c.commit()
+                            _c.close()
+                            _sum_existing.add(message_id)
+                            _summaries_created += 1
+                            _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                            _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
                     except Exception as e:
                         _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                         _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
@@ -468,14 +511,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     if context_snippets:
                         sys_prompt += "\n\nRELEVANT CONTEXT FROM PAST EMAILS AND CONTACTS:\n" + "\n\n---\n\n".join(context_snippets[:5])
                     try:
-                        reply = await llm_call_async(
-                            url=url, model=model,
+                        reply = await task_llm_call_async(
                             messages=[
                                 {"role": "system", "content": sys_prompt},
                                 {"role": "user", "content": f"Original email:\nFrom: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\nDraft a reply. Return only the reply body text."},
                             ],
-                            temperature=0.7, max_tokens=1024,
-                            headers=req_headers, timeout=90,
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.7, max_tokens=1024, timeout=90,
                         )
                         reply = _apply_email_style_mechanics(_extract_reply(reply or ""))
                         if reply:
@@ -502,6 +545,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                 # ── Calendar event extraction (independent of reply drafting) ──
                 if need_cal:
                     _cal_run_count = 0
+                    _cal_event_uids = []
+                    _cal_parse_ok = False
                     try:
                         # Pull a snapshot of upcoming events so the LLM can decide
                         # create vs update vs cancel based on what already exists.
@@ -510,8 +555,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                         _existing_summary = get_upcoming_events(_acct_owner, horizon_days=60, limit=40)
                         existing_json = json.dumps(_existing_summary)
                         is_sent = _folder.lower().startswith("sent") or "sent" in _folder.lower()
-                        cal_extract = await llm_call_async(
-                            url=url, model=model,
+                        cal_extract = await task_llm_call_async(
                             messages=[
                                 {"role": "system", "content": (
                                     "You are a calendar assistant. The user receives emails AND sends replies "
@@ -562,8 +606,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                     f"{body[:4000]}"
                                 )},
                             ],
-                            temperature=0.1, max_tokens=16384,
-                            headers=req_headers, timeout=180,
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.1, max_tokens=16384, timeout=75,
                         )
                         _raw_original = cal_extract or ""
                         cal_extract = _strip_think(_raw_original)
@@ -573,10 +618,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             if matches:
                                 cal_extract = matches[-1].group()
                         logger.info(f"[cal-extract] uid={uid.decode() if isinstance(uid, bytes) else uid} folder={_folder} subj={subject[:50]!r} raw_len={len(cal_extract)} orig_len={len(_raw_original)} raw={cal_extract[:800]!r}")
-                        jm = re.search(r'\[.*\]', cal_extract, re.DOTALL)
-                        if jm:
+                        ops = _extract_json_array_from_text(cal_extract)
+                        if ops is not None:
                             try:
-                                ops = json.loads(jm.group())
+                                _cal_parse_ok = True
                                 logger.info(f"[cal-extract] parsed {len(ops)} op(s)")
                                 if isinstance(ops, list) and ops:
                                     from src.tool_implementations import do_manage_calendar
@@ -606,6 +651,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                             r = await do_manage_calendar(json.dumps(args), owner=_acct_owner)
                                             if r.get("exit_code", 0) == 0:
                                                 logger.info(f"[cal-extract] Updated event uid={cuid} → {op.get('title')} {op['date']}")
+                                                if cuid and cuid not in _cal_event_uids:
+                                                    _cal_event_uids.append(cuid)
                                                 _cal_run_count += 1
                                             else:
                                                 logger.warning(f"[cal-extract] update failed: {r.get('error')}")
@@ -686,29 +733,41 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                             r = await do_manage_calendar(cal_args, owner=_acct_owner)
                                             if r.get("exit_code", 0) == 0:
                                                 logger.info(f"[cal-extract] Created event: {op['title']} on {op['date']}")
+                                                _created_uid = (r.get("uid") or "").strip()
+                                                if _created_uid and _created_uid not in _cal_event_uids:
+                                                    _cal_event_uids.append(_created_uid)
                                                 _events_created += 1
                                                 _cal_run_count += 1
                                             else:
                                                 logger.warning(f"[cal-extract] create failed: {r.get('error')} args={cal_args[:200]}")
                             except Exception as je:
                                 logger.warning(f"[cal-extract] JSON parse failed: {je} on raw={cal_extract[:200]!r}")
+                        else:
+                            logger.warning(f"[cal-extract] no JSON array found on raw={cal_extract[:200]!r}")
                     except Exception as e:
                         logger.warning(f"[cal-extract] Meeting extraction LLM call failed for uid={uid}: {e}")
                     else:
-                        # Record we processed this email so we don't re-LLM next run.
-                        # Only mark as processed on success ? transient LLM failures
-                        # are retried on the next poll run (matches summary/reply pattern).
+                        # Record successfully parsed results so we don't re-LLM
+                        # no-op emails. Transient LLM failures are retried on
+                        # the next poll run.
                         try:
-                            _cc = _sql3.connect(SCHEDULED_DB)
-                            _cc.execute(
-                                "INSERT OR REPLACE INTO email_calendar_extractions "
-                                "(message_id, owner, uid, events_created, created_at) VALUES (?, ?, ?, ?, ?)",
-                                (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid),
-                                 _cal_run_count, datetime.utcnow().isoformat())
-                            )
-                            _cc.commit()
-                            _cc.close()
-                            _cal_existing.add(message_id)
+                            if _cal_parse_ok:
+                                _cc = _sql3.connect(SCHEDULED_DB)
+                                _cc.execute(
+                                    "INSERT OR REPLACE INTO email_calendar_extractions "
+                                    "(message_id, owner, uid, event_uids, events_created, created_at) VALUES (?, ?, ?, ?, ?, ?)",
+                                    (
+                                        message_id,
+                                        account_owner or "",
+                                        uid.decode() if isinstance(uid, bytes) else str(uid),
+                                        json.dumps(_cal_event_uids),
+                                        _cal_run_count,
+                                        datetime.utcnow().isoformat(),
+                                    ),
+                                )
+                                _cc.commit()
+                                _cc.close()
+                                _cal_existing.add(message_id)
                         except Exception as ce:
                             logger.debug(f"Could not cache calendar extraction: {ce}")
 
@@ -742,9 +801,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             "temperature": 0,
                             tok_key: 200,
                         }
-                        urg_raw = await llm_call_async(
-                            url=url, model=model, messages=payload["messages"],
-                            temperature=0, max_tokens=200, headers=req_headers, timeout=60,
+                        urg_raw = await task_llm_call_async(
+                            messages=payload["messages"],
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0, max_tokens=200, timeout=60,
                         )
                         urg_raw = _strip_think(urg_raw or "")
                         urg_raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", urg_raw, flags=re.MULTILINE).strip()
@@ -845,8 +906,13 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                         class_sys = (
                             "Classify the email. Return ONLY a JSON object, no prose, no markdown fences. "
                             "Schema: {\"tags\": [\"tag1\"], \"spam\": false, \"reason\": \"short\"}. "
-                            "Pick 1-2 tags from: work, personal, finance, bills, receipt, travel, "
-                            "newsletter, promo, notification, security, social, shopping, calendar.\n\n"
+                            "Pick 1-3 tags from: work, personal, urgent, action-needed, finance, bills, "
+                            "receipt, legal, travel, newsletter, promo, notification, security, social, "
+                            "shopping, calendar, support.\n\n"
+                            "Use work for professional/company/client/operations messages. "
+                            "Use personal for friends/family/private-life messages. "
+                            "Use urgent for real time-sensitive consequences. "
+                            "Use action-needed when the user likely needs to reply, pay, sign, book, or decide.\n\n"
                             "Set spam=true for ANY of:\n"
                             "- Phishing, scams, chain mail, deceptive offers\n"
                             "- Marketing/promotional blasts (\"special offer\", \"limited time\", discount codes)\n"
@@ -863,70 +929,55 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             "If it's a mass-mailed generic update with no personal CTA, mark spam=true even if from a legitimate service. "
                             "Reason should be 5-10 words."
                         )
-                        tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
-                        payload = {
-                            "model": model,
-                            "messages": [
+                        raw_out = await task_llm_call_async(
+                            messages=[
                                 {"role": "system", "content": class_sys},
                                 {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body[:4000]}"},
                             ],
-                            tok_key: 512,
-                            "temperature": 0.1,
-                            "stream": False,
-                        }
-                        # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
-                        if _restricts_temperature(model):
-                            payload.pop("temperature", None)
-                        # to_thread keeps the event loop responsive during the LLM call
-                        resp = await asyncio.to_thread(
-                            _req.post, url, json=payload, headers=req_headers, timeout=120
+                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            owner=account_owner or None,
+                            temperature=0.1, max_tokens=512, timeout=120,
                         )
-                        if not resp.ok:
-                            logger.warning(f"Auto-classify {uid.decode() if isinstance(uid, bytes) else str(uid)} HTTP {resp.status_code}: {resp.text[:200]}")
-                        else:
-                            rdata = resp.json()
-                            m = (rdata.get("choices") or [{}])[0].get("message", {})
-                            raw_out = (m.get("content") or "").strip()
-                            raw_out = _strip_think(raw_out)
-                            raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
-                            jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
-                            parsed = None
-                            if jm:
-                                try:
-                                    parsed = json.loads(jm.group(0))
-                                except Exception:
-                                    parsed = None
-                            if parsed is not None:
-                                _ALLOWED_TAGS = {"work","personal","finance","bills","receipt","travel",
-                                                 "newsletter","marketing","notification","security","social",
-                                                 "shopping","calendar"}
-                                raw_tags = parsed.get("tags") or []
-                                if isinstance(raw_tags, str):
-                                    raw_tags = [raw_tags]
-                                tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
-                                tags = ["marketing" if t == "promo" else t for t in tags]
-                                tags = [t for t in tags if t in _ALLOWED_TAGS][:2]
-                                is_spam = bool(parsed.get("spam"))
-                                spam_reason = str(parsed.get("reason") or "")[:200]
+                        raw_out = _strip_think((raw_out or "").strip())
+                        raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
+                        jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
+                        parsed = None
+                        if jm:
+                            try:
+                                parsed = json.loads(jm.group(0))
+                            except Exception:
+                                parsed = None
+                        if parsed is not None:
+                            _ALLOWED_TAGS = {"work","personal","urgent","action-needed","finance","bills",
+                                             "receipt","legal","travel","newsletter","marketing","notification",
+                                             "security","social","shopping","calendar","support"}
+                            raw_tags = parsed.get("tags") or []
+                            if isinstance(raw_tags, str):
+                                raw_tags = [raw_tags]
+                            tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
+                            tags = ["marketing" if t == "promo" else t for t in tags]
+                            tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
+                            is_spam = bool(parsed.get("spam"))
+                            spam_reason = str(parsed.get("reason") or "")[:200]
 
-                                moved_to = ""
-                                if is_spam and auto_spam and spam_folder:
-                                    if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
-                                        moved_to = spam_folder
-                                        logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
+                            moved_to = ""
+                            if is_spam and auto_spam and spam_folder:
+                                if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
+                                    moved_to = spam_folder
+                                    logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
 
-                                _c = _sql3.connect(SCHEDULED_DB)
-                                _c.execute("""
-                                    INSERT OR REPLACE INTO email_tags
-                                    (message_id, owner, uid, folder, subject, sender, tags, spam_verdict,
-                                     spam_reason, moved_to, model_used, created_at)
-                                    VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?, ?, ?)
-                                """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), subject, sender,
-                                      json.dumps(tags), 1 if is_spam else 0,
-                                      spam_reason, moved_to, model, datetime.utcnow().isoformat()))
-                                _c.commit()
-                                _c.close()
-                                _tag_existing.add(message_id)
+                            _c = _sql3.connect(SCHEDULED_DB)
+                            _c.execute("""
+                                INSERT OR REPLACE INTO email_tags
+                                (message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict,
+                                 spam_reason, moved_to, model_used, created_at)
+                                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """, (message_id, account_owner or "", account_id or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender,
+                                  json.dumps(tags), 1 if is_spam else 0,
+                                  spam_reason, moved_to, model, datetime.utcnow().isoformat()))
+                            _c.commit()
+                            _c.close()
+                            _tag_existing.add(message_id)
                     except Exception as e:
                         logger.warning(f"Auto-classify {uid} failed: {e}")
 

routes/gallery/gallery_helpers.py

@@ -99,6 +99,7 @@ def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any
         "filename": img.filename,
         "url": f"/api/generated-image/{img.filename}",
         "prompt": img.prompt,
+        "caption": img.caption or "",
         "model": img.model,
         "size": img.size,
         "quality": img.quality,

routes/gallery/gallery_routes.py

@@ -77,6 +77,39 @@ def _normalize_image_endpoint_base(url: str) -> str:
     return base
 
 
+def _is_openai_api_base(url: str) -> bool:
+    """Return True only when url's hostname is exactly api.openai.com."""
+    from urllib.parse import urlsplit
+    try:
+        candidate = url if "://" in url else f"https://{url}"
+        return urlsplit(candidate).hostname == "api.openai.com"
+    except Exception:
+        return False
+
+
+_GALLERY_ENDPOINT_PATHS = frozenset({
+    "/images/edits",
+    "/images/generations",
+    "/images/harmonize",
+    "/images/img2img",
+    "/images/inpaint",
+    "/images/upscale",
+    "/images/variations",
+    "/sdapi/v1/img2img",
+})
+
+
+def _join_checked_gallery_endpoint(base: str, path: str) -> str:
+    """Append a known-constant gallery path suffix to a validated base URL.
+
+    Rejects paths not in the pre-approved list so arbitrary strings can never
+    be spliced into the URL passed to httpx.
+    """
+    if path not in _GALLERY_ENDPOINT_PATHS:
+        raise ValueError(f"Unexpected gallery path: {path!r}")
+    return base + path
+
+
 def _visible_image_endpoint_query(db, owner: str | None):
     from src.auth_helpers import owner_filter
     q = db.query(ModelEndpoint).filter(
@@ -255,9 +288,10 @@ def setup_gallery_routes() -> APIRouter:
                 pass
             try:
                 db.commit()
-            except Exception as e:
+            except Exception:
                 db.rollback()
-                raise HTTPException(500, f"DB commit failed: {e}")
+                logger.exception("gallery_replace: DB commit failed")
+                raise HTTPException(500, "Image update failed")
             return {"ok": True, "width": img.width, "height": img.height}
         finally:
             db.close()
@@ -385,8 +419,9 @@ def setup_gallery_routes() -> APIRouter:
                     return {"image": data.get("data", [{}])[0].get("b64_json", "")}
                 # Fallback: no upscale endpoint — return error
                 return {"error": f"Upscale endpoint not available ({resp.status_code})"}
-        except Exception as e:
-            return {"error": str(e)}
+        except Exception:
+            logger.exception("ai_upscale: request failed")
+            return {"error": "Upscale request failed"}
 
     # ---- POST /api/gallery/style-transfer ----
     @router.post("/api/gallery/style-transfer")
@@ -431,8 +466,9 @@ def setup_gallery_routes() -> APIRouter:
                     if img_data:
                         return {"image": img_data}
                 return {"error": f"Style transfer failed ({resp.status_code})"}
-        except Exception as e:
-            return {"error": str(e)}
+        except Exception:
+            logger.exception("style_transfer: request failed")
+            return {"error": "Style transfer failed"}
 
     # ---- GET /api/gallery/tags ----
     @router.get("/api/gallery/tags")
@@ -588,9 +624,9 @@ def setup_gallery_routes() -> APIRouter:
                 "tags": sorted(all_tags),
                 "models": all_models,
             }
-        except Exception as e:
-            logger.error(f"Failed to fetch gallery library: {e}")
-            raise HTTPException(500, f"Failed to fetch gallery library: {e}")
+        except Exception:
+            logger.exception("Failed to fetch gallery library")
+            raise HTTPException(500, "Failed to fetch gallery library")
         finally:
             db.close()
 
@@ -766,9 +802,10 @@ def setup_gallery_routes() -> APIRouter:
             return _image_to_dict(img)
         except HTTPException:
             raise
-        except Exception as e:
+        except Exception:
             db.rollback()
-            raise HTTPException(500, str(e))
+            logger.exception("patch_gallery_image: update failed")
+            raise HTTPException(500, "Image update failed")
         finally:
             db.close()
 
@@ -845,9 +882,10 @@ def setup_gallery_routes() -> APIRouter:
                     cleared += 1
             db.commit()
             return {"ok": True, "cleared": cleared}
-        except Exception as e:
+        except Exception:
             db.rollback()
-            raise HTTPException(500, str(e))
+            logger.exception("clear_gallery_user_tags: failed")
+            raise HTTPException(500, "Tag update failed")
         finally:
             db.close()
 
@@ -871,9 +909,10 @@ def setup_gallery_routes() -> APIRouter:
                     cleared += 1
             db.commit()
             return {"ok": True, "cleared": cleared}
-        except Exception as e:
+        except Exception:
             db.rollback()
-            raise HTTPException(500, str(e))
+            logger.exception("clear_gallery_ai_tags: failed")
+            raise HTTPException(500, "Tag update failed")
         finally:
             db.close()
 
@@ -909,9 +948,10 @@ def setup_gallery_routes() -> APIRouter:
                     img.tags = ', '.join(cleaned)
             db.commit()
             return {"ok": True, "rows_touched": rows_touched, "tags_removed": tags_removed}
-        except Exception as e:
+        except Exception:
             db.rollback()
-            raise HTTPException(500, str(e))
+            logger.exception("dedupe_gallery_tags: failed")
+            raise HTTPException(500, "Tag deduplication failed")
         finally:
             db.close()
 
@@ -1029,9 +1069,10 @@ def setup_gallery_routes() -> APIRouter:
             return {"status": "deleted", "id": image_id}
         except HTTPException:
             raise
-        except Exception as e:
+        except Exception:
             db.rollback()
-            raise HTTPException(500, str(e))
+            logger.exception("delete_gallery_image: failed")
+            raise HTTPException(500, "Image deletion failed")
         finally:
             db.close()
 
@@ -1044,21 +1085,22 @@ def setup_gallery_routes() -> APIRouter:
         import httpx
         user = require_privilege(request, "can_generate_images")
         body = await request.json()
-        # Use endpoint from request body (editor dropdown) or fall back to DB lookup
-        base = (body.pop("_endpoint", "") or "").rstrip("/")
+        # Use endpoint from request body (editor dropdown) or fall back to DB lookup.
+        # Store as requested_base to avoid carrying user input into the outbound request.
+        requested_base = (body.pop("_endpoint", "") or "").rstrip("/")
         # SSRF hardening: validate a client-supplied endpoint before any
         # outbound request (mirrors routes/embedding_routes.py).
-        if base:
+        if requested_base:
             from src.url_safety import check_outbound_url
             ok, reason = check_outbound_url(
-                base,
+                requested_base,
                 block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true",
             )
             if not ok:
                 raise HTTPException(400, f"Rejected endpoint URL: {reason}")
         chosen_model = (body.pop("_model", "") or "").strip()
         api_key = None
-        if not base:
+        if not requested_base:
             db = SessionLocal()
             try:
                 ep = _first_visible_image_endpoint(db, user)
@@ -1069,32 +1111,23 @@ def setup_gallery_routes() -> APIRouter:
             finally:
                 db.close()
         else:
-            # Pull api_key from the matching DB row so OpenAI auth works.
-            # Users may have stored base_url with/without /v1 suffix and with/without
-            # trailing slash, so compare normalized forms.
-            def _norm_url(u: str) -> str:
-                if not u:
-                    return u
-                u = u.rstrip("/")
-                if u.endswith("/v1"):
-                    u = u[:-3]
-                return u
-            _target = _norm_url(base)
+            # Resolve the client-supplied base to a registered visible endpoint.
+            # Admins are not exempted — gallery proxy routes must use a DB row
+            # so the outbound URL never depends directly on request-body input.
             db = SessionLocal()
             try:
-                ep = _visible_image_endpoint_for_base(db, _target, user)
-                if ep:
-                    base = (ep.base_url or base).rstrip("/")
-                    api_key = ep.api_key
-                elif user and not _current_user_is_admin(request, user):
+                ep = _visible_image_endpoint_for_base(db, requested_base, user)
+                if not ep:
                     raise HTTPException(403, "Choose a registered image endpoint")
+                base = ep.base_url.rstrip("/")
+                api_key = ep.api_key
             finally:
                 db.close()
 
         if not base.endswith("/v1"):
             base += "/v1"
 
-        is_openai = "api.openai.com" in base
+        is_openai = _is_openai_api_base(base)
 
         if is_openai:
             # OpenAI path: /v1/images/edits with gpt-image-1.
@@ -1131,8 +1164,9 @@ def setup_gallery_routes() -> APIRouter:
                 mask_buf.seek(0)
             except HTTPException:
                 raise
-            except Exception as e:
-                raise HTTPException(400, f"Failed to prepare OpenAI request: {e}")
+            except Exception:
+                logger.exception("inpaint_proxy: failed to prepare OpenAI request")
+                raise HTTPException(400, "Failed to prepare inpaint request")
 
             width = int(body.get("width") or 1024)
             height = int(body.get("height") or 1024)
@@ -1163,9 +1197,10 @@ def setup_gallery_routes() -> APIRouter:
             headers = {"Authorization": f"Bearer {api_key}"}
             try:
                 async with httpx.AsyncClient(timeout=120) as client:
-                    r = await client.post(f"{base}/images/edits", headers=headers, data=data, files=files)
+                    r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), headers=headers, data=data, files=files)
                     if r.status_code != 200:
-                        raise HTTPException(r.status_code, f"OpenAI edit failed: {r.text[:300]}")
+                        logger.error("inpaint_proxy OpenAI edit: status %s", r.status_code)
+                        raise HTTPException(r.status_code, "OpenAI edit failed")
                     result = r.json()
                     raw_b64 = None
                     if result.get("data"):
@@ -1212,16 +1247,18 @@ def setup_gallery_routes() -> APIRouter:
             if chosen_model:
                 body["model"] = chosen_model
             async with httpx.AsyncClient(timeout=120) as client:
-                r = await client.post(f"{base}/images/inpaint", json=body)
+                r = await client.post(_join_checked_gallery_endpoint(base, "/images/inpaint"), json=body)
                 if r.status_code != 200:
-                    raise HTTPException(r.status_code, f"Inpaint failed: {r.text[:200]}")
+                    logger.error("inpaint_proxy diffusion: status %s", r.status_code)
+                    raise HTTPException(r.status_code, "Inpaint request failed")
                 return r.json()
         except httpx.TimeoutException:
             raise HTTPException(504, "Inpaint request timed out (120s)")
         except HTTPException:
             raise
-        except Exception as e:
-            raise HTTPException(502, f"Inpaint error: {str(e)}")
+        except Exception:
+            logger.exception("inpaint_proxy: request failed")
+            raise HTTPException(502, "Inpaint request failed")
 
     # ---- POST /api/image/harmonize — proper img2img call ----
     # Earlier version routed through inpaint with a full-white mask, but
@@ -1243,24 +1280,23 @@ def setup_gallery_routes() -> APIRouter:
         if not image_b64:
             raise HTTPException(400, "No image provided")
 
-        endpoint = (body.get("_endpoint") or "").rstrip("/")
+        requested_base = (body.get("_endpoint") or "").rstrip("/")
         # SSRF hardening: a client-supplied endpoint is fetched server-side
         # below, so validate it first (mirrors routes/embedding_routes.py).
         # Local-first means loopback/LAN is allowed by default; the cloud
         # metadata range and non-HTTP(S) schemes are always rejected.
-        if endpoint:
+        if requested_base:
             from src.url_safety import check_outbound_url
             ok, reason = check_outbound_url(
-                endpoint,
+                requested_base,
                 block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true",
             )
             if not ok:
                 raise HTTPException(400, f"Rejected endpoint URL: {reason}")
         model = (body.get("_model") or "").strip()
 
-        base = endpoint
         api_key = None
-        if not base:
+        if not requested_base:
             db = SessionLocal()
             try:
                 ep = _first_visible_image_endpoint(db, user)
@@ -1271,14 +1307,16 @@ def setup_gallery_routes() -> APIRouter:
             finally:
                 db.close()
         else:
+            # Resolve the client-supplied base to a registered visible endpoint.
+            # Admins are not exempted — gallery proxy routes must use a DB row
+            # so the outbound URL never depends directly on request-body input.
             db = SessionLocal()
             try:
-                ep = _visible_image_endpoint_for_base(db, base, user)
-                if ep:
-                    base = (ep.base_url or base).rstrip("/")
-                    api_key = ep.api_key
-                elif user and not _current_user_is_admin(request, user):
+                ep = _visible_image_endpoint_for_base(db, requested_base, user)
+                if not ep:
                     raise HTTPException(403, "Choose a registered image endpoint")
+                base = ep.base_url.rstrip("/")
+                api_key = ep.api_key
             finally:
                 db.close()
 
@@ -1313,7 +1351,7 @@ def setup_gallery_routes() -> APIRouter:
         # source. Earlier hack (alpha-blend the regen back at `strength`)
         # produced visibly broken results, so we refuse and tell the
         # user to spin up a real diffusion endpoint instead.
-        if "api.openai.com" in base:
+        if _is_openai_api_base(base):
             raise HTTPException(400,
                 "Harmonize needs a diffusion server that supports img2img "
                 "(SD WebUI / Forge / Comfy). OpenAI's API doesn't expose "
@@ -1378,14 +1416,16 @@ def setup_gallery_routes() -> APIRouter:
         # 1024×1024 inference pass on slower setups.
         async with httpx.AsyncClient(timeout=240) as client:
             for path, kind, payload in candidates:
-                target = base_root + path if path.startswith("/sdapi") else base + path
+                _effective_base = base_root if path.startswith("/sdapi") else base
+                target = _join_checked_gallery_endpoint(_effective_base, path)
                 try:
                     r = await client.post(target, json=payload, headers=headers)
                     if r.status_code == 404:
                         last_err = f"{path}: 404"
                         continue  # try next variant
                     if r.status_code != 200:
-                        last_err = f"{path}: {r.status_code} {r.text[:120]}"
+                        logger.warning("harmonize: %s returned %s", path, r.status_code)
+                        last_err = f"{path}: {r.status_code}"
                         continue
                     data = r.json()
                     # Normalise return shape.
@@ -1394,8 +1434,8 @@ def setup_gallery_routes() -> APIRouter:
                         # surface it now instead of trying the other routes
                         # (otherwise the real error gets buried under 404s).
                         if data.get("error") and not data.get("image"):
-                            raise HTTPException(502,
-                                f"Diffusion server error at {path}: {data['error']}")
+                            logger.warning("harmonize: server error at %s: %s", path, data.get("error"))
+                            raise HTTPException(502, f"Diffusion server error at {path}")
                         if data.get("image"):
                             return {"image": data["image"]}
                         if data.get("images") and isinstance(data["images"], list):
@@ -1415,15 +1455,15 @@ def setup_gallery_routes() -> APIRouter:
                                 if img_b64:
                                     return {"image": img_b64}
                     last_err = f"{path}: server returned no image"
-                except httpx.ConnectError as e:
-                    raise HTTPException(502, f"Can't reach diffusion server at {base}: {e}")
+                except httpx.ConnectError:
+                    logger.warning("harmonize: can't reach diffusion server at %s", base)
+                    raise HTTPException(502, "Can't reach diffusion server")
                 except httpx.TimeoutException:
                     raise HTTPException(504, "Harmonize timed out (240s) — restart the diffusion server or lower Color match / disable Seam fix")
         raise HTTPException(502,
-            f"None of the img2img routes worked on {base}. "
-            f"Last response: {last_err or 'unknown'}. "
-            "Your diffusion server needs to expose one of /v1/images/harmonize, "
-            "/v1/images/img2img, /v1/images/variations, or /sdapi/v1/img2img.")
+            "No supported img2img route responded. "
+            "Your diffusion server needs to expose one of: "
+            "/v1/images/harmonize, /v1/images/img2img, /v1/images/variations, /sdapi/v1/img2img.")
 
     # ---- POST /api/image/sharpen ----
     @router.post("/api/image/sharpen")
@@ -1467,8 +1507,8 @@ def setup_gallery_routes() -> APIRouter:
             import base64, io
             from PIL import Image
             import numpy as np
-        except ImportError as e:
-            raise HTTPException(500, f"Server missing dependency: {e}")
+        except ImportError:
+            raise HTTPException(500, "Server missing a required dependency")
         # Decode source image (RGB; Real-ESRGAN doesn't preserve alpha).
         img_bytes = base64.b64decode(image_b64)
         src = Image.open(io.BytesIO(img_bytes)).convert("RGB")
@@ -1495,9 +1535,9 @@ def setup_gallery_routes() -> APIRouter:
             buf = io.BytesIO()
             out_img.save(buf, format="PNG")
             return {"image": base64.b64encode(buf.getvalue()).decode()}
-        except Exception as e:
-            logger.warning(f"Denoise failed: {e}")
-            return {"error": f"Denoise failed: {e}"}
+        except Exception:
+            logger.warning("Denoise failed", exc_info=True)
+            return {"error": "Denoise failed"}
 
     # ---- POST /api/image/upscale-local ----
     # Local Real-ESRGAN upscale (2× or 4×). Self-contained — no diffusion
@@ -1518,8 +1558,8 @@ def setup_gallery_routes() -> APIRouter:
             import base64, io
             from PIL import Image
             import numpy as np
-        except ImportError as e:
-            raise HTTPException(500, f"Server missing dependency: {e}")
+        except ImportError:
+            raise HTTPException(500, "Server missing a required dependency")
         img_bytes = base64.b64decode(image_b64)
         src = Image.open(io.BytesIO(img_bytes)).convert("RGB")
         try:
@@ -1543,9 +1583,9 @@ def setup_gallery_routes() -> APIRouter:
             buf = io.BytesIO()
             out_img.save(buf, format="PNG")
             return {"image": base64.b64encode(buf.getvalue()).decode()}
-        except Exception as e:
-            logger.warning(f"Upscale failed: {e}")
-            return {"error": f"Upscale failed: {e}"}
+        except Exception:
+            logger.warning("AI upscale failed", exc_info=True)
+            return {"error": "AI upscale failed"}
 
     # ---- POST /api/image/remove-bg ----
     @router.post("/api/image/remove-bg")
@@ -1703,8 +1743,9 @@ def setup_gallery_routes() -> APIRouter:
             buf = io.BytesIO()
             enhanced.save(buf, format="PNG")
             return {"image": base64.b64encode(buf.getvalue()).decode(), "method": "pil"}
-        except Exception as e:
-            raise HTTPException(500, f"Face enhancement failed: {str(e)}")
+        except Exception:
+            logger.exception("enhance_face: failed")
+            raise HTTPException(500, "Face enhancement failed")
 
     # ---- Album management (path-param routes) ----
 
@@ -1899,9 +1940,8 @@ def setup_gallery_routes() -> APIRouter:
             async with httpx.AsyncClient(timeout=60) as client:
                 resp = await client.post(chat_url, json=payload, headers=h)
                 if resp.status_code != 200:
-                    body = resp.text[:500]
-                    logger.error(f"Vision model {resp.status_code}: {body}")
-                    return {"error": f"Vision model returned {resp.status_code}: {body[:200]}"}
+                    logger.error("ai_tag vision model: status %s: %s", resp.status_code, resp.text[:500])
+                    return {"error": "Vision model request failed"}
                 data = resp.json()
                 # Anthropic returns content[0].text, OpenAI returns choices[0].message.content
                 if provider == "anthropic":
@@ -1917,9 +1957,9 @@ def setup_gallery_routes() -> APIRouter:
             return {"ok": True, "ai_tags": tag_str}
         except HTTPException:
             raise
-        except Exception as e:
-            logger.error(f"AI tagging failed: {e}")
-            return {"error": str(e)}
+        except Exception:
+            logger.exception("AI tagging failed")
+            return {"error": "Auto-tagging failed"}
         finally:
             db.close()
 

routes/gallery_helpers.py

@@ -1,9 +1,9 @@
-"""Backward-compat shim — canonical location is routes/gallery/gallery_helpers.py.
+"""Backward-compat shim - canonical location is routes/gallery/gallery_helpers.py.
 
 This module is replaced in ``sys.modules`` by the canonical module object so
 that ``import routes.gallery_helpers``, ``from routes.gallery_helpers import X``,
 ``importlib.import_module("routes.gallery_helpers")``, and
-``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the *same*
+``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the same
 object. Keeps existing import paths working after slice 2a (#4082/#4071).
 """
 

routes/gallery_routes.py

@@ -1,9 +1,9 @@
-"""Backward-compat shim — canonical location is routes/gallery/gallery_routes.py.
+"""Backward-compat shim - canonical location is routes/gallery/gallery_routes.py.
 
 This module is replaced in ``sys.modules`` by the canonical module object so
 that ``import routes.gallery_routes``, ``from routes.gallery_routes import X``,
 ``importlib.import_module("routes.gallery_routes")``, and
-``monkeypatch.setattr(routes.gallery_routes, ...)`` all operate on the *same*
+``monkeypatch.setattr(routes.gallery_routes, ...)`` all operate on the same
 object the application actually uses. Keeps existing import paths working
 after slice 2a (#4082/#4071). Source-introspection tests read the canonical
 file by path.

routes/history_routes.py

@@ -3,7 +3,8 @@
 import json
 import uuid
 import logging
-from typing import Dict, Any
+import re
+from typing import Dict, Any, Optional
 
 from fastapi import APIRouter, Request, HTTPException
 
@@ -19,6 +20,63 @@ from routes.session_routes import (
 
 logger = logging.getLogger(__name__)
 
+_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
+_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
+
+
+def _history_display_content(content: Any) -> Any:
+    """Return a lightweight browser-display copy of stored message content.
+
+    Older multimodal user messages may be persisted as a JSON *string*
+    containing image_url blocks with inline base64 image bytes. Those bytes are
+    needed for model calls when the turn is first sent, but they should not be
+    sent back through /api/history every time the user opens the chat. The
+    attachment metadata already carries file ids/names for the UI cards.
+    """
+    if isinstance(content, list):
+        text_parts = []
+        omitted_media = 0
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "text":
+                text = block.get("text")
+                if isinstance(text, str) and text:
+                    text_parts.append(text)
+            elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
+                omitted_media += 1
+        text = "\n".join(text_parts).strip()
+        if omitted_media and not text:
+            return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
+        return text
+
+    if not isinstance(content, str):
+        return content
+    if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
+        return content
+
+    stripped = content.lstrip()
+    if stripped.startswith("["):
+        try:
+            blocks = json.loads(content)
+        except (json.JSONDecodeError, TypeError, ValueError):
+            blocks = None
+        if isinstance(blocks, list):
+            text_parts = []
+            for block in blocks:
+                if not isinstance(block, dict):
+                    continue
+                if block.get("type") == "text":
+                    text = block.get("text")
+                    if isinstance(text, str) and text:
+                        text_parts.append(text)
+            if text_parts:
+                return "\n".join(text_parts).strip()
+
+    if "data:image/" in content:
+        return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
+    return content
+
 
 def _merge_continue_rows_to_delete(db_messages, db1, db2):
     """DB rows to delete when merging the last two assistant messages.
@@ -43,9 +101,69 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2):
 def setup_history_routes(session_manager) -> APIRouter:
     router = APIRouter(tags=["history"])
 
+    def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
+        entry = {"role": m.role, "content": _history_display_content(m.content)}
+        meta = {}
+        if m.meta_data:
+            try:
+                meta = json.loads(m.meta_data) or {}
+            except (json.JSONDecodeError, ValueError):
+                meta = {}
+        if m.timestamp and "timestamp" not in meta:
+            meta["timestamp"] = m.timestamp.isoformat() + "Z"
+        if meta:
+            entry["metadata"] = meta
+        return entry
+
     @router.get("/api/history/{session_id}")
-    async def get_session_history(request: Request, session_id: str) -> Dict[str, Any]:
+    async def get_session_history(
+        request: Request,
+        session_id: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Dict[str, Any]:
         _verify_session_owner(request, session_id)
+        if limit is not None:
+            page_limit = max(1, min(int(limit), 100))
+            db = SessionLocal()
+            try:
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session is None:
+                    raise HTTPException(404, f"Session '{session_id}' not found")
+
+                total = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .count()
+                )
+                page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
+                page_offset = max(0, min(page_offset, total))
+                rows = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .offset(page_offset)
+                    .limit(page_limit)
+                    .all()
+                )
+                history_dict = [
+                    entry for entry in (_db_history_entry(m) for m in rows)
+                    if not (entry.get("metadata") or {}).get("hidden")
+                ]
+                return {
+                    "history": history_dict,
+                    "model": db_session.model,
+                    "endpoint_url": db_session.endpoint_url,
+                    "name": db_session.name,
+                    "offset": page_offset,
+                    "limit": page_limit,
+                    "total": total,
+                    "has_more_before": page_offset > 0,
+                    "has_more_after": page_offset + len(rows) < total,
+                }
+            finally:
+                db.close()
+
         try:
             session = session_manager.get_session(session_id)
         except KeyError:
@@ -57,7 +175,7 @@ def setup_history_routes(session_manager) -> APIRouter:
                 # Skip hidden messages (e.g. compaction summaries for AI context)
                 if msg.metadata and msg.metadata.get("hidden"):
                     continue
-                entry = {"role": msg.role, "content": msg.content}
+                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
                 if msg.metadata:
                     entry["metadata"] = msg.metadata
                 history_dict.append(entry)
@@ -66,7 +184,7 @@ def setup_history_routes(session_manager) -> APIRouter:
                     continue
                 entry = {
                     "role": msg.get("role", ""),
-                    "content": msg.get("content", ""),
+                    "content": _history_display_content(msg.get("content", "")),
                 }
                 if msg.get("metadata"):
                     entry["metadata"] = msg["metadata"]
@@ -82,21 +200,9 @@ def setup_history_routes(session_manager) -> APIRouter:
                     .order_by(DbChatMessage.timestamp)
                     .all()
                 )
-                import json as _json
                 db_history = []
                 for m in db_messages:
-                    entry = {"role": m.role, "content": m.content}
-                    meta = {}
-                    if m.meta_data:
-                        try:
-                            meta = _json.loads(m.meta_data) or {}
-                        except (json.JSONDecodeError, ValueError):
-                            meta = {}
-                    if m.timestamp and "timestamp" not in meta:
-                        meta["timestamp"] = m.timestamp.isoformat() + "Z"
-                    if meta:
-                        entry["metadata"] = meta
-                    db_history.append(entry)
+                    db_history.append(_db_history_entry(m))
                 if db_history:
                     # Rebuild in-memory history from the full set so hidden
                     # messages (e.g. compaction summaries) are kept for AI context.

routes/memory/__init__.py

@@ -0,0 +1,5 @@
+"""Memory route domain package (slice 2c, #4082/#4071).
+
+Contains memory_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/memory_routes.py re-exports from here.
+"""

routes/memory/memory_routes.py

@@ -0,0 +1,552 @@
+# routes/memory_routes.py
+from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
+from typing import Dict, Any, Optional, List
+import json
+import os
+import re
+import tempfile
+import time
+from datetime import datetime
+import logging
+
+# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
+# Strips one prefix per call so import-from-LLM-output doesn't leave the
+# numbering inside the saved memory text. Bullet markers (-, *, •) are
+# also peeled here for the same reason.
+_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
+
+
+def _strip_list_prefix(text: str) -> str:
+    if not text:
+        return text
+    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
+
+from services.memory import MemoryManager
+from core.session_manager import SessionManager
+from src.request_models import MemoryAddRequest
+from core.database import SessionLocal
+from src.llm_core import llm_call_async
+from services.memory.memory_extractor import audit_memories
+from src.auth_helpers import get_current_user, require_user
+from src.endpoint_resolver import resolve_endpoint
+from src.task_endpoint import resolve_task_endpoint
+from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
+
+logger = logging.getLogger(__name__)
+
+
+def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
+    """Set up memory-related routes."""
+    router = APIRouter(prefix="/api/memory", tags=["memory"])
+
+    def _owner(request: Request) -> Optional[str]:
+        return get_current_user(request)
+
+    def _assert_session_owner(session_obj, user):
+        """SECURITY: 404 if the caller does not own this session.
+
+        SessionManager.get_session is NOT owner-scoped — it returns any
+        session by id. These routes accept a caller-supplied session id, so
+        without this gate a user could target another tenant's session and
+        leak their chat history, their session-scoped LLM credentials, or the
+        session title. Mirrors session_routes / webhook_routes ownership.
+        """
+        if user is not None and getattr(session_obj, "owner", None) != user:
+            raise HTTPException(404, "Session not found")
+
+    def _verify_memory_owner(memory: dict, user: Optional[str]):
+        """Raise 404 if user doesn't own this memory.
+
+        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
+        allowed any user to read/edit/delete memories with an empty/null owner
+        field, which leaked legacy data across the multi-user deploy.
+        """
+        if user is None:
+            return  # Auth disabled
+        if memory.get("owner") != user:
+            raise HTTPException(404, "Memory not found")
+
+    @router.post("/debug")
+    def debug_memory_relevance(request: Request, query: str = Form(...)):
+        """Debug which memories would be triggered for a query"""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
+
+        return {
+            "query": query,
+            "total_memories": len(memories),
+            "relevant_count": len(relevant),
+            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
+                                 for m in relevant]
+        }
+
+    @router.post("/add", response_model=Dict[str, Any])
+    async def api_add_memory(
+        request: Request,
+        memory_data: Optional[MemoryAddRequest] = None
+    ):
+        """Add a new memory entry with optional category, source, and session reference."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+        if memory_data is None:
+            form = await request.form()
+            memory_data = MemoryAddRequest(
+                text=form.get("text"),
+                category=form.get("category", "fact"),
+                source=form.get("source", "user"),
+                session_id=form.get("session_id")
+            )
+
+        user = _owner(request)
+        text = (memory_data.text or "").strip()
+        if not text:
+            raise HTTPException(400, "empty memory")
+        user_mem = memory_manager.load(owner=user)
+        if memory_manager.find_duplicates(text, user_mem):
+            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
+
+        if memory_data.session_id:
+            try:
+                session_obj = session_manager.get_session(memory_data.session_id)
+            except KeyError:
+                raise HTTPException(404, "Session not found")
+            _assert_session_owner(session_obj, user)
+
+        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
+        if memory_data.session_id:
+            new_entry["session_id"] = memory_data.session_id
+        all_mem = memory_manager.load_all()
+        all_mem.append(new_entry)
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.add(new_entry["id"], text)
+        try:
+            from src.event_bus import fire_event
+            fire_event("memory_added", user)
+        except Exception:
+            logger.debug("memory_added event dispatch failed", exc_info=True)
+        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
+
+    @router.get("")
+    def api_get_memory(request: Request):
+        """Return all memory entries with their metadata."""
+        user = _owner(request)
+        return {"memory": memory_manager.load(owner=user)}
+
+    @router.post("/search")
+    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
+        """Search across all memories with optional filters."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+
+        if session_id:
+            memories = [m for m in memories if m.get("session_id") == session_id]
+
+        if category:
+            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
+
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+
+        return {"memories": relevant, "total": len(relevant), "query": query}
+
+    @router.get("/timeline")
+    def memory_timeline(request: Request):
+        """Get memories in chronological order with source session information."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        results = []
+        for memory in sorted_memories:
+            if "timestamp" in memory:
+                try:
+                    dt = datetime.fromtimestamp(memory["timestamp"])
+                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
+                except (ValueError, OSError, OverflowError):
+                    memory["timestamp_str"] = "Unknown"
+            else:
+                memory["timestamp_str"] = "Unknown"
+
+            session_id = memory.get("session_id")
+            if session_id and session_id in session_manager.sessions:
+                try:
+                    session = session_manager.get_session(session_id)
+                    if session:
+                        _assert_session_owner(session, user)
+                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
+                except KeyError:
+                    memory["session_name"] = "Unknown"
+                except HTTPException as exc:
+                    if exc.status_code != 404:
+                        raise
+                    memory["session_name"] = "Unknown"
+            else:
+                memory["session_name"] = "Unknown"
+
+            results.append(memory)
+
+        return {"timeline": results, "total": len(results)}
+
+    @router.get("/by-session/{session_id}")
+    def get_memory_by_session(request: Request, session_id: str):
+        """Get all memories associated with a specific session."""
+        user = _owner(request)
+        try:
+            _session_obj = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, f"Session {session_id} not found")
+        _assert_session_owner(_session_obj, user)
+        memories = memory_manager.load(owner=user)
+        session_memories = [m for m in memories if m.get("session_id") == session_id]
+
+        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        try:
+            session = session_manager.get_session(session_id)
+            session_name = session.name if session else f"Session {session_id[:6]}"
+        except KeyError:
+            session_name = f"Session {session_id[:6]}"
+
+        for memory in session_memories:
+            memory["session_name"] = session_name
+
+        return {
+            "session_id": session_id,
+            "session_name": session_name,
+            "memory_count": len(session_memories),
+            "memories": session_memories
+        }
+
+    @router.post("/extract")
+    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
+        """Analyze a session's chat history and return memory suggestions."""
+        require_user(request)
+        try:
+            sess = session_manager.get_session(session)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        _assert_session_owner(sess, _owner(request))
+
+        system_msg = {
+            "role": "system",
+            "content": (
+                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
+                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
+                "might want to remember for future interactions. Return each piece of information as a JSON object "
+                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
+                "Only include information that is specific and likely to be useful later."
+            ),
+        }
+        messages = [system_msg] + sess.get_context_messages()
+
+        t_url, t_model, t_headers = resolve_task_endpoint(
+            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
+        )
+
+        try:
+            suggestion_text = await llm_call_async(
+                t_url,
+                t_model,
+                messages,
+                temperature=0.2,
+                max_tokens=500,
+                headers=t_headers,
+            )
+            try:
+                suggestions = json.loads(suggestion_text)
+                if isinstance(suggestions, list):
+                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
+                else:
+                    suggestions = []
+            except json.JSONDecodeError:
+                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
+
+            return {"suggestions": [s for s in suggestions if s]}
+        except Exception as e:
+            logger.error(f"LLM memory extraction failed (session {session}): {e}")
+            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
+            return {"suggestions": [item["text"] for item in fallback]}
+
+    @router.post("/audit")
+    async def api_audit_memories(request: Request, session: str = Form(None)):
+        """Deduplicate and consolidate memories via LLM.
+
+        Uses task/utility/default settings through the shared resolver, with
+        the active session as fallback when no task or utility model is set.
+        Returns before and after memory counts.
+        """
+        user = _owner(request)
+        fallback_url = fallback_model = None
+        fallback_headers = None
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+                fallback_url = sess.endpoint_url
+                fallback_model = sess.model
+                fallback_headers = sess.headers
+            except KeyError:
+                pass
+
+        endpoint_url, model, headers = resolve_task_endpoint(
+            fallback_url, fallback_model, fallback_headers, owner=user
+        )
+
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No default model configured — set one in Settings")
+
+        result = await audit_memories(
+            memory_manager,
+            memory_vector,
+            endpoint_url,
+            model,
+            headers,
+            owner=user,
+        )
+
+        if "error" in result and "before" not in result:
+            raise HTTPException(502, f"Audit failed: {result['error']}")
+
+        return {
+            "ok": "error" not in result,
+            "before": result.get("before", 0),
+            "after": result.get("after", 0),
+            "removed": result.get("before", 0) - result.get("after", 0),
+            # True when the audit skipped the LLM because nothing changed
+            # since the last tidy. Frontend already says "Already clean"
+            # for removed==0, so this is here for future use / debugging.
+            "already_tidy": bool(result.get("already_tidy")),
+        }
+
+    @router.post("/import")
+    async def import_memories_from_file(
+        request: Request,
+        session: str | None = Form(None),
+        file: UploadFile = File(...)
+    ):
+        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+
+        endpoint_url = None
+        model = None
+        headers = {}
+
+        user = _owner(request)
+
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+            except KeyError:
+                sess = None
+            except HTTPException as exc:
+                if exc.status_code != 404:
+                    raise
+                sess = None
+
+            if sess is None:
+                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
+                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
+            else:
+                endpoint_url, model, headers = resolve_task_endpoint(
+                    sess.endpoint_url, sess.model, sess.headers, owner=user
+                )
+        else:
+            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
+    
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
+
+        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
+        filename = file.filename or "upload"
+        _, ext = os.path.splitext(filename.lower())
+
+        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
+        if ext not in allowed:
+            raise HTTPException(400, f"Unsupported file type: {ext}")
+
+        # Extract text based on file type
+        if ext == ".pdf":
+            from src.document_processor import _process_pdf
+            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+                tmp.write(content)
+                tmp_path = tmp.name
+            try:
+                text = _process_pdf(tmp_path, owner=_owner(request))
+            finally:
+                os.unlink(tmp_path)
+        else:
+            try:
+                text = content.decode("utf-8")
+            except UnicodeDecodeError:
+                from charset_normalizer import detect
+                encoding = (detect(content) or {}).get("encoding") or "utf-8"
+                text = content.decode(encoding, errors="replace")
+
+        if not text.strip():
+            return {"suggestions": [], "message": "No readable content found"}
+
+        # Fast path: a .json upload that already looks like a memories export
+        # (list of {text, category, ...} dicts, or list of strings) round-trips
+        # directly without spending an LLM call to re-extract its own output.
+        # Without this, re-importing a memories.json from another account
+        # ran the file through the extractor, which often re-emitted the
+        # entries as a numbered list (and the numbering leaked into the
+        # `text` field).
+        if ext == ".json":
+            try:
+                parsed = json.loads(text)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list) and parsed:
+                direct = []
+                for item in parsed:
+                    if isinstance(item, dict) and item.get("text"):
+                        direct.append({
+                            "text": _strip_list_prefix(str(item["text"])),
+                            "category": item.get("category") or "fact",
+                        })
+                    elif isinstance(item, str) and item.strip():
+                        direct.append({
+                            "text": _strip_list_prefix(item.strip()),
+                            "category": "fact",
+                        })
+                if direct:
+                    return {"suggestions": direct, "filename": filename}
+
+        # Truncate very long documents
+        if len(text) > 15000:
+            text = text[:15000] + "\n[Truncated]"
+
+        # Send to LLM for memory extraction
+        import_prompt = (
+            "You are a memory extraction assistant. The user uploaded a document. "
+            "Analyze the text below and extract specific, useful facts — things like "
+            "names, preferences, jobs, locations, relationships, opinions, projects, "
+            "goals, contacts, or any other personal details worth remembering.\n\n"
+            "Rules:\n"
+            "- Each fact should be a short, self-contained statement\n"
+            "- Do NOT extract generic knowledge\n"
+            "- Focus on personal, memorable information\n"
+            "- If there are no useful facts, return an empty array\n\n"
+            "Return a JSON array of objects with 'text' and 'category' fields.\n"
+            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
+            "Return ONLY valid JSON, no markdown fences."
+        )
+
+        try:
+            raw = await llm_call_async(
+                endpoint_url,
+                model,
+                [
+                    {"role": "system", "content": import_prompt},
+                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
+                ],
+                temperature=0.2,
+                max_tokens=2000,
+                headers=headers,
+            )
+
+            # Parse JSON
+            raw = raw.strip()
+            if raw.startswith("```"):
+                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+
+            suggestions = json.loads(raw)
+            if isinstance(suggestions, list):
+                normalized = []
+                for s in suggestions:
+                    if not s:
+                        continue
+                    if isinstance(s, dict):
+                        s = dict(s)
+                        if s.get("text"):
+                            s["text"] = _strip_list_prefix(str(s["text"]))
+                        normalized.append(s)
+                    else:
+                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
+                suggestions = normalized
+            else:
+                suggestions = []
+
+            return {"suggestions": suggestions, "filename": filename}
+
+        except json.JSONDecodeError:
+            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
+            # numbering the model added so saved memories don't keep the prefix.
+            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
+            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
+        except Exception as e:
+            logger.error(f"Memory import extraction failed: {e}")
+            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
+
+    @router.post("/{memory_id}/pin")
+    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
+        """Pin or unpin a memory. Pinned memories are always included in context."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["pinned"] = pinned
+                memory_manager.save(all_mem)
+                return {"ok": True, "pinned": pinned}
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
+    @router.get("/{memory_id}")
+    def get_memory_item(request: Request, memory_id: str):
+        """Get a specific memory item by ID."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        for memory in memories:
+            if memory["id"] == memory_id:
+                return {"memory": memory}
+
+        raise HTTPException(404, "Memory not found")
+
+    @router.put("/{memory_id}")
+    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
+        """Update an existing memory item with new text and optional category."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["text"] = text.strip()
+                if category:
+                    all_mem[i]["category"] = category
+                all_mem[i]["timestamp"] = int(time.time())
+
+                memory_manager.save(all_mem)
+                # Sync vector index (remove old, add updated)
+                if memory_vector and memory_vector.healthy:
+                    memory_vector.remove(memory_id)
+                    memory_vector.add(memory_id, text.strip())
+                return {"ok": True, "message": "Memory updated successfully"}
+
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    @router.delete("/{memory_id}")
+    def delete_memory(request: Request, memory_id: str):
+        """Delete a memory item by its ID."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+
+        # Find and verify ownership before deleting
+        target = next((m for m in all_mem if m["id"] == memory_id), None)
+        if not target:
+            raise HTTPException(404, f"Memory item {memory_id} not found")
+        _verify_memory_owner(target, user)
+
+        all_mem = [m for m in all_mem if m["id"] != memory_id]
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.remove(memory_id)
+        return {"ok": True, "message": "Memory deleted successfully"}
+
+    return router

routes/memory_routes.py

@@ -1,552 +1,18 @@
-# routes/memory_routes.py
-from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
-from typing import Dict, Any, Optional, List
-import json
-import os
-import re
-import tempfile
-import time
-from datetime import datetime
-import logging
+"""Backward-compat shim — canonical location is routes/memory/memory_routes.py.
 
-# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
-# Strips one prefix per call so import-from-LLM-output doesn't leave the
-# numbering inside the saved memory text. Bullet markers (-, *, •) are
-# also peeled here for the same reason.
-_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.memory_routes``, ``from routes.memory_routes import X``,
+``importlib.import_module("routes.memory_routes")``, and
+``monkeypatch.setattr(routes.memory_routes, "ATTR", ...)`` (used by
+test_memory_routes_session_owner.py and test_memory_owner_isolation.py via
+``import ... as mr`` + ``setattr(mr, ...)``) all operate on the *same* object
+the application actually uses. Keeps existing import paths working after
+slice 2c (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
+import sys as _sys
 
-def _strip_list_prefix(text: str) -> str:
-    if not text:
-        return text
-    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
+from routes.memory import memory_routes as _canonical  # noqa: F401
 
-from services.memory import MemoryManager
-from core.session_manager import SessionManager
-from src.request_models import MemoryAddRequest
-from core.database import SessionLocal
-from src.llm_core import llm_call_async
-from services.memory.memory_extractor import audit_memories
-from src.auth_helpers import get_current_user, require_user
-from src.endpoint_resolver import resolve_endpoint
-from src.task_endpoint import resolve_task_endpoint
-from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
-
-logger = logging.getLogger(__name__)
-
-
-def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
-    """Set up memory-related routes."""
-    router = APIRouter(prefix="/api/memory", tags=["memory"])
-
-    def _owner(request: Request) -> Optional[str]:
-        return get_current_user(request)
-
-    def _assert_session_owner(session_obj, user):
-        """SECURITY: 404 if the caller does not own this session.
-
-        SessionManager.get_session is NOT owner-scoped — it returns any
-        session by id. These routes accept a caller-supplied session id, so
-        without this gate a user could target another tenant's session and
-        leak their chat history, their session-scoped LLM credentials, or the
-        session title. Mirrors session_routes / webhook_routes ownership.
-        """
-        if user is not None and getattr(session_obj, "owner", None) != user:
-            raise HTTPException(404, "Session not found")
-
-    def _verify_memory_owner(memory: dict, user: Optional[str]):
-        """Raise 404 if user doesn't own this memory.
-
-        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
-        allowed any user to read/edit/delete memories with an empty/null owner
-        field, which leaked legacy data across the multi-user deploy.
-        """
-        if user is None:
-            return  # Auth disabled
-        if memory.get("owner") != user:
-            raise HTTPException(404, "Memory not found")
-
-    @router.post("/debug")
-    def debug_memory_relevance(request: Request, query: str = Form(...)):
-        """Debug which memories would be triggered for a query"""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
-
-        return {
-            "query": query,
-            "total_memories": len(memories),
-            "relevant_count": len(relevant),
-            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
-                                 for m in relevant]
-        }
-
-    @router.post("/add", response_model=Dict[str, Any])
-    async def api_add_memory(
-        request: Request,
-        memory_data: Optional[MemoryAddRequest] = None
-    ):
-        """Add a new memory entry with optional category, source, and session reference."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-        if memory_data is None:
-            form = await request.form()
-            memory_data = MemoryAddRequest(
-                text=form.get("text"),
-                category=form.get("category", "fact"),
-                source=form.get("source", "user"),
-                session_id=form.get("session_id")
-            )
-
-        user = _owner(request)
-        text = (memory_data.text or "").strip()
-        if not text:
-            raise HTTPException(400, "empty memory")
-        user_mem = memory_manager.load(owner=user)
-        if memory_manager.find_duplicates(text, user_mem):
-            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
-
-        if memory_data.session_id:
-            try:
-                session_obj = session_manager.get_session(memory_data.session_id)
-            except KeyError:
-                raise HTTPException(404, "Session not found")
-            _assert_session_owner(session_obj, user)
-
-        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
-        if memory_data.session_id:
-            new_entry["session_id"] = memory_data.session_id
-        all_mem = memory_manager.load_all()
-        all_mem.append(new_entry)
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.add(new_entry["id"], text)
-        try:
-            from src.event_bus import fire_event
-            fire_event("memory_added", user)
-        except Exception:
-            logger.debug("memory_added event dispatch failed", exc_info=True)
-        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
-
-    @router.get("")
-    def api_get_memory(request: Request):
-        """Return all memory entries with their metadata."""
-        user = _owner(request)
-        return {"memory": memory_manager.load(owner=user)}
-
-    @router.post("/search")
-    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
-        """Search across all memories with optional filters."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-
-        if session_id:
-            memories = [m for m in memories if m.get("session_id") == session_id]
-
-        if category:
-            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
-
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
-
-        return {"memories": relevant, "total": len(relevant), "query": query}
-
-    @router.get("/timeline")
-    def memory_timeline(request: Request):
-        """Get memories in chronological order with source session information."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        results = []
-        for memory in sorted_memories:
-            if "timestamp" in memory:
-                try:
-                    dt = datetime.fromtimestamp(memory["timestamp"])
-                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
-                except (ValueError, OSError, OverflowError):
-                    memory["timestamp_str"] = "Unknown"
-            else:
-                memory["timestamp_str"] = "Unknown"
-
-            session_id = memory.get("session_id")
-            if session_id and session_id in session_manager.sessions:
-                try:
-                    session = session_manager.get_session(session_id)
-                    if session:
-                        _assert_session_owner(session, user)
-                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
-                except KeyError:
-                    memory["session_name"] = "Unknown"
-                except HTTPException as exc:
-                    if exc.status_code != 404:
-                        raise
-                    memory["session_name"] = "Unknown"
-            else:
-                memory["session_name"] = "Unknown"
-
-            results.append(memory)
-
-        return {"timeline": results, "total": len(results)}
-
-    @router.get("/by-session/{session_id}")
-    def get_memory_by_session(request: Request, session_id: str):
-        """Get all memories associated with a specific session."""
-        user = _owner(request)
-        try:
-            _session_obj = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session {session_id} not found")
-        _assert_session_owner(_session_obj, user)
-        memories = memory_manager.load(owner=user)
-        session_memories = [m for m in memories if m.get("session_id") == session_id]
-
-        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        try:
-            session = session_manager.get_session(session_id)
-            session_name = session.name if session else f"Session {session_id[:6]}"
-        except KeyError:
-            session_name = f"Session {session_id[:6]}"
-
-        for memory in session_memories:
-            memory["session_name"] = session_name
-
-        return {
-            "session_id": session_id,
-            "session_name": session_name,
-            "memory_count": len(session_memories),
-            "memories": session_memories
-        }
-
-    @router.post("/extract")
-    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
-        """Analyze a session's chat history and return memory suggestions."""
-        require_user(request)
-        try:
-            sess = session_manager.get_session(session)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        _assert_session_owner(sess, _owner(request))
-
-        system_msg = {
-            "role": "system",
-            "content": (
-                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
-                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
-                "might want to remember for future interactions. Return each piece of information as a JSON object "
-                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
-                "Only include information that is specific and likely to be useful later."
-            ),
-        }
-        messages = [system_msg] + sess.get_context_messages()
-
-        t_url, t_model, t_headers = resolve_task_endpoint(
-            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
-        )
-
-        try:
-            suggestion_text = await llm_call_async(
-                t_url,
-                t_model,
-                messages,
-                temperature=0.2,
-                max_tokens=500,
-                headers=t_headers,
-            )
-            try:
-                suggestions = json.loads(suggestion_text)
-                if isinstance(suggestions, list):
-                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
-                else:
-                    suggestions = []
-            except json.JSONDecodeError:
-                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
-
-            return {"suggestions": [s for s in suggestions if s]}
-        except Exception as e:
-            logger.error(f"LLM memory extraction failed (session {session}): {e}")
-            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
-            return {"suggestions": [item["text"] for item in fallback]}
-
-    @router.post("/audit")
-    async def api_audit_memories(request: Request, session: str = Form(None)):
-        """Deduplicate and consolidate memories via LLM.
-
-        Uses task/utility/default settings through the shared resolver, with
-        the active session as fallback when no task or utility model is set.
-        Returns before and after memory counts.
-        """
-        user = _owner(request)
-        fallback_url = fallback_model = None
-        fallback_headers = None
-        if session:
-            try:
-                sess = session_manager.get_session(session)
-                _assert_session_owner(sess, user)
-                fallback_url = sess.endpoint_url
-                fallback_model = sess.model
-                fallback_headers = sess.headers
-            except KeyError:
-                pass
-
-        endpoint_url, model, headers = resolve_task_endpoint(
-            fallback_url, fallback_model, fallback_headers, owner=user
-        )
-
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No default model configured — set one in Settings")
-
-        result = await audit_memories(
-            memory_manager,
-            memory_vector,
-            endpoint_url,
-            model,
-            headers,
-            owner=user,
-        )
-
-        if "error" in result and "before" not in result:
-            raise HTTPException(502, f"Audit failed: {result['error']}")
-
-        return {
-            "ok": "error" not in result,
-            "before": result.get("before", 0),
-            "after": result.get("after", 0),
-            "removed": result.get("before", 0) - result.get("after", 0),
-            # True when the audit skipped the LLM because nothing changed
-            # since the last tidy. Frontend already says "Already clean"
-            # for removed==0, so this is here for future use / debugging.
-            "already_tidy": bool(result.get("already_tidy")),
-        }
-
-    @router.post("/import")
-    async def import_memories_from_file(
-        request: Request,
-        session: str | None = Form(None),
-        file: UploadFile = File(...)
-    ):
-        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-
-        endpoint_url = None
-        model = None
-        headers = {}
-
-        user = _owner(request)
-
-        if session:
-            try:
-                sess = session_manager.get_session(session)
-                _assert_session_owner(sess, user)
-            except KeyError:
-                sess = None
-            except HTTPException as exc:
-                if exc.status_code != 404:
-                    raise
-                sess = None
-
-            if sess is None:
-                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
-                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
-            else:
-                endpoint_url, model, headers = resolve_task_endpoint(
-                    sess.endpoint_url, sess.model, sess.headers, owner=user
-                )
-        else:
-            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
-    
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
-
-        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
-        filename = file.filename or "upload"
-        _, ext = os.path.splitext(filename.lower())
-
-        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
-        if ext not in allowed:
-            raise HTTPException(400, f"Unsupported file type: {ext}")
-
-        # Extract text based on file type
-        if ext == ".pdf":
-            from src.document_processor import _process_pdf
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
-                tmp.write(content)
-                tmp_path = tmp.name
-            try:
-                text = _process_pdf(tmp_path, owner=_owner(request))
-            finally:
-                os.unlink(tmp_path)
-        else:
-            try:
-                text = content.decode("utf-8")
-            except UnicodeDecodeError:
-                from charset_normalizer import detect
-                encoding = (detect(content) or {}).get("encoding") or "utf-8"
-                text = content.decode(encoding, errors="replace")
-
-        if not text.strip():
-            return {"suggestions": [], "message": "No readable content found"}
-
-        # Fast path: a .json upload that already looks like a memories export
-        # (list of {text, category, ...} dicts, or list of strings) round-trips
-        # directly without spending an LLM call to re-extract its own output.
-        # Without this, re-importing a memories.json from another account
-        # ran the file through the extractor, which often re-emitted the
-        # entries as a numbered list (and the numbering leaked into the
-        # `text` field).
-        if ext == ".json":
-            try:
-                parsed = json.loads(text)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list) and parsed:
-                direct = []
-                for item in parsed:
-                    if isinstance(item, dict) and item.get("text"):
-                        direct.append({
-                            "text": _strip_list_prefix(str(item["text"])),
-                            "category": item.get("category") or "fact",
-                        })
-                    elif isinstance(item, str) and item.strip():
-                        direct.append({
-                            "text": _strip_list_prefix(item.strip()),
-                            "category": "fact",
-                        })
-                if direct:
-                    return {"suggestions": direct, "filename": filename}
-
-        # Truncate very long documents
-        if len(text) > 15000:
-            text = text[:15000] + "\n[Truncated]"
-
-        # Send to LLM for memory extraction
-        import_prompt = (
-            "You are a memory extraction assistant. The user uploaded a document. "
-            "Analyze the text below and extract specific, useful facts — things like "
-            "names, preferences, jobs, locations, relationships, opinions, projects, "
-            "goals, contacts, or any other personal details worth remembering.\n\n"
-            "Rules:\n"
-            "- Each fact should be a short, self-contained statement\n"
-            "- Do NOT extract generic knowledge\n"
-            "- Focus on personal, memorable information\n"
-            "- If there are no useful facts, return an empty array\n\n"
-            "Return a JSON array of objects with 'text' and 'category' fields.\n"
-            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
-            "Return ONLY valid JSON, no markdown fences."
-        )
-
-        try:
-            raw = await llm_call_async(
-                endpoint_url,
-                model,
-                [
-                    {"role": "system", "content": import_prompt},
-                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
-                ],
-                temperature=0.2,
-                max_tokens=2000,
-                headers=headers,
-            )
-
-            # Parse JSON
-            raw = raw.strip()
-            if raw.startswith("```"):
-                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-
-            suggestions = json.loads(raw)
-            if isinstance(suggestions, list):
-                normalized = []
-                for s in suggestions:
-                    if not s:
-                        continue
-                    if isinstance(s, dict):
-                        s = dict(s)
-                        if s.get("text"):
-                            s["text"] = _strip_list_prefix(str(s["text"]))
-                        normalized.append(s)
-                    else:
-                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
-                suggestions = normalized
-            else:
-                suggestions = []
-
-            return {"suggestions": suggestions, "filename": filename}
-
-        except json.JSONDecodeError:
-            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
-            # numbering the model added so saved memories don't keep the prefix.
-            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
-            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
-        except Exception as e:
-            logger.error(f"Memory import extraction failed: {e}")
-            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
-
-    @router.post("/{memory_id}/pin")
-    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
-        """Pin or unpin a memory. Pinned memories are always included in context."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["pinned"] = pinned
-                memory_manager.save(all_mem)
-                return {"ok": True, "pinned": pinned}
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
-    @router.get("/{memory_id}")
-    def get_memory_item(request: Request, memory_id: str):
-        """Get a specific memory item by ID."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        for memory in memories:
-            if memory["id"] == memory_id:
-                return {"memory": memory}
-
-        raise HTTPException(404, "Memory not found")
-
-    @router.put("/{memory_id}")
-    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
-        """Update an existing memory item with new text and optional category."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["text"] = text.strip()
-                if category:
-                    all_mem[i]["category"] = category
-                all_mem[i]["timestamp"] = int(time.time())
-
-                memory_manager.save(all_mem)
-                # Sync vector index (remove old, add updated)
-                if memory_vector and memory_vector.healthy:
-                    memory_vector.remove(memory_id)
-                    memory_vector.add(memory_id, text.strip())
-                return {"ok": True, "message": "Memory updated successfully"}
-
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    @router.delete("/{memory_id}")
-    def delete_memory(request: Request, memory_id: str):
-        """Delete a memory item by its ID."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-
-        # Find and verify ownership before deleting
-        target = next((m for m in all_mem if m["id"] == memory_id), None)
-        if not target:
-            raise HTTPException(404, f"Memory item {memory_id} not found")
-        _verify_memory_owner(target, user)
-
-        all_mem = [m for m in all_mem if m["id"] != memory_id]
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.remove(memory_id)
-        return {"ok": True, "message": "Memory deleted successfully"}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/model_routes.py

@@ -19,6 +19,7 @@ from fastapi.responses import StreamingResponse
 from core.database import SessionLocal, ModelEndpoint, Session as DbSession
 from core.log_safety import redact_url as _redact_url_for_log
 from core.middleware import require_admin
+from src.constants import COOKBOOK_STATE_FILE
 from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
 from src.tls_overrides import llm_verify
 from src.settings import load_settings as _load_settings, save_settings as _save_settings
@@ -112,6 +113,67 @@ def _clear_endpoint_settings_for_endpoint(settings: dict, ep_id: str, *, include
     return cleared
 
 
+_COOKBOOK_ACTIVE_SERVE_STATUSES = {
+    "starting", "loading", "ready", "running", "restarting",
+}
+
+
+def _active_cookbook_endpoint_ids() -> set[str]:
+    """Endpoint IDs owned by active Cookbook serve tasks.
+
+    Cookbook auto-registers endpoints with ids like ``local-*``. Those rows are
+    managed lifecycle state, not durable user configuration. If a tmux stream is
+    stopped or an old task lingers, the row must stop participating in model
+    selection and defaults.
+    """
+    try:
+        if not os.path.exists(COOKBOOK_STATE_FILE):
+            return set()
+        with open(COOKBOOK_STATE_FILE, "r", encoding="utf-8") as fh:
+            raw = fh.read()
+        state = json.loads(raw)
+    except Exception:
+        return set()
+    out: set[str] = set()
+    for task in state.get("tasks") or []:
+        if not isinstance(task, dict) or task.get("type") != "serve":
+            continue
+        if str(task.get("status") or "").lower() not in _COOKBOOK_ACTIVE_SERVE_STATUSES:
+            continue
+        ep_id = task.get("_endpointId") or task.get("endpointId") or task.get("endpoint_id")
+        if ep_id:
+            out.add(str(ep_id))
+    return out
+
+
+def _disable_stale_cookbook_local_endpoints(db) -> int:
+    """Disable enabled cookbook endpoints whose serve task is no longer active."""
+    active_ids = _active_cookbook_endpoint_ids()
+    if not active_ids:
+        return 0
+    stale = (
+        db.query(ModelEndpoint)
+        .filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+        .filter(ModelEndpoint.id.like("local-%"))
+        .filter(~ModelEndpoint.id.in_(active_ids))
+        .all()
+    )
+    if not stale:
+        return 0
+    settings = _load_settings()
+    touched_settings = False
+    for ep in stale:
+        ep.is_enabled = False
+        ep.model_refresh_mode = "disabled"
+        if _clear_endpoint_settings_for_endpoint(settings, ep.id):
+            touched_settings = True
+        logger.info("Disabled stale Cookbook endpoint %s (%s @ %s)", ep.id, ep.name, ep.base_url)
+    if touched_settings:
+        _save_settings(settings)
+    db.commit()
+    return len(stale)
+
+
 def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     """Remove endpoint references from scoped or legacy-flat user preferences."""
     if not isinstance(all_prefs, dict):
@@ -125,7 +187,24 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     return cleared_users
 
 
-def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool:
+def _endpoint_visible_model_ids(ep: Any) -> List[str]:
+    """Known visible model ids for an endpoint, including pinned/manual ids."""
+    if ep is None:
+        return []
+    return _visible_models(
+        getattr(ep, "cached_models", None),
+        getattr(ep, "hidden_models", None),
+        getattr(ep, "pinned_models", None),
+    )
+
+
+def _default_endpoint_needs_assignment(
+    current_default_id: str,
+    enabled_endpoint_ids,
+    *,
+    current_default_endpoint: Any = None,
+    current_default_model: str = "",
+) -> bool:
     """Whether the global default chat endpoint should be (re)assigned.
 
     True when nothing is configured yet, or the configured default no longer
@@ -137,7 +216,14 @@ def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint
     """
     if not current_default_id:
         return True
-    return current_default_id not in enabled_endpoint_ids
+    if current_default_id not in enabled_endpoint_ids:
+        return True
+    if current_default_endpoint is None:
+        return False
+    if not (current_default_model or "").strip():
+        return True
+    visible = _endpoint_visible_model_ids(current_default_endpoint)
+    return bool(visible and current_default_model not in visible)
 
 
 # Loopback hosts a user might type for a local model server (LM Studio,
@@ -1194,6 +1280,8 @@ def setup_model_routes(model_discovery):
                 db = SessionLocal()
                 changed = False
                 try:
+                    if _disable_stale_cookbook_local_endpoints(db):
+                        changed = True
                     endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
                     now = _time.time()
                     groups: Dict[str, Dict[str, Any]] = {}
@@ -1267,6 +1355,8 @@ def setup_model_routes(model_discovery):
 
         db = SessionLocal()
         try:
+            if _disable_stale_cookbook_local_endpoints(db):
+                _invalidate_models_cache()
             q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
             if owner and not is_admin:
                 # Regular users see: their own endpoints + null-owner
@@ -1336,7 +1426,7 @@ def setup_model_routes(model_discovery):
         return {"hosts": [], "items": items}
 
     @router.get("/models")
-    def api_models(request: Request, refresh: bool = False):
+    def api_models(request: Request, refresh: bool = False, background: bool = True):
         """Get available models — per-user (caller sees only their endpoints +
         legacy/shared null-owner rows). Cached per-user for 30s."""
         # Require auth; "" is the unconfigured single-user mode, treated as
@@ -1378,8 +1468,11 @@ def setup_model_routes(model_discovery):
             return cache_entry["data"]
         result = _fetch_models(owner=owner, is_admin=_is_admin)
         _models_cache[_cache_key] = {"data": result, "time": now}
-        # Kick off background refresh to update caches from live endpoints
-        _refresh_caches_bg(force=refresh)
+        # Kick off background refresh to update caches from live endpoints.
+        # Page boot can opt out with background=false so opening Odysseus does
+        # not start endpoint probes against slow/offline model servers.
+        if background or refresh:
+            _refresh_caches_bg(force=refresh)
         return result
 
     # Brief cache for local-probe results so picker-open doesn't hammer
@@ -1388,6 +1481,7 @@ def setup_model_routes(model_discovery):
     # within ~8s of the user noticing.
     _LOCAL_PROBE_TTL = 8.0
     _local_probe_cache: Dict[str, Any] = {"data": None, "time": 0.0}
+    _local_probe_inflight: Dict[str, Any] = {"task": None}
 
     @router.get("/model-endpoints/probe-local")
     async def probe_local_endpoints(request: Request):
@@ -1402,58 +1496,72 @@ def setup_model_routes(model_discovery):
                 (now - _local_probe_cache["time"]) < _LOCAL_PROBE_TTL):
             return _local_probe_cache["data"]
 
-        db = SessionLocal()
-        try:
-            endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
-            local_eps = []
-            for ep in endpoints:
-                base = _normalize_base(ep.base_url)
-                kind = _effective_endpoint_kind(ep, base)
-                if _classify_endpoint(base, kind) == "local":
-                    local_eps.append((ep.id, base, ep.api_key))
-        finally:
-            db.close()
-
-        grouped: Dict[str, Dict[str, Any]] = {}
-        for ep_id, base, api_key in local_eps:
-            key = _refresh_key(base, api_key)
-            grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
-
-        async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
-            t0 = _time.time()
-            try:
-                import asyncio as _asyncio
-                # Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
-                # local vLLM endpoints on Tailscale links where the model
-                # server is still loading (Qwen3.5-122B takes 2–3 min to
-                # warm); /v1/models can take 500–2500 ms on a busy box,
-                # which pushed _ping_endpoint's full path-discovery sweep
-                # past the cap and marked the row offline despite the
-                # user actively chatting with it.
-                ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
-                lat = round((_time.time() - t0) * 1000)
-                return {
-                    "alive": bool(ping.get("reachable")),
-                    "latency_ms": lat,
-                    "status_code": ping.get("status_code"),
-                    "error": ping.get("error"),
-                }
-            except Exception as e:
-                return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
-
         import asyncio as _asyncio
-        results_list = await _asyncio.gather(
-            *[_probe_one(data) for data in grouped.values()],
-            return_exceptions=False,
-        )
-        results: Dict[str, Any] = {}
-        for data, r in zip(grouped.values(), results_list):
-            for eid in data["endpoint_ids"]:
-                results[eid] = r
+        task = _local_probe_inflight.get("task")
+        if task is not None and not task.done():
+            return await task
 
-        _local_probe_cache["data"] = results
-        _local_probe_cache["time"] = now
-        return results
+        async def _compute_local_probe() -> Dict[str, Any]:
+            db = SessionLocal()
+            try:
+                if _disable_stale_cookbook_local_endpoints(db):
+                    _invalidate_models_cache()
+                endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
+                local_eps = []
+                for ep in endpoints:
+                    base = _normalize_base(ep.base_url)
+                    kind = _effective_endpoint_kind(ep, base)
+                    if _classify_endpoint(base, kind) == "local":
+                        local_eps.append((ep.id, base, ep.api_key))
+            finally:
+                db.close()
+
+            grouped: Dict[str, Dict[str, Any]] = {}
+            for ep_id, base, api_key in local_eps:
+                key = _refresh_key(base, api_key)
+                grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
+
+            async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
+                t0 = _time.time()
+                try:
+                    # Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
+                    # local vLLM endpoints on Tailscale links where the model
+                    # server is still loading (Qwen3.5-122B takes 2–3 min to
+                    # warm); /v1/models can take 500–2500 ms on a busy box,
+                    # which pushed _ping_endpoint's full path-discovery sweep
+                    # past the cap and marked the row offline despite the
+                    # user actively chatting with it.
+                    ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
+                    lat = round((_time.time() - t0) * 1000)
+                    return {
+                        "alive": bool(ping.get("reachable")),
+                        "latency_ms": lat,
+                        "status_code": ping.get("status_code"),
+                        "error": ping.get("error"),
+                    }
+                except Exception as e:
+                    return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
+
+            results_list = await _asyncio.gather(
+                *[_probe_one(data) for data in grouped.values()],
+                return_exceptions=False,
+            )
+            results: Dict[str, Any] = {}
+            for data, r in zip(grouped.values(), results_list):
+                for eid in data["endpoint_ids"]:
+                    results[eid] = r
+
+            _local_probe_cache["data"] = results
+            _local_probe_cache["time"] = _time.time()
+            return results
+
+        task = _asyncio.create_task(_compute_local_probe())
+        _local_probe_inflight["task"] = task
+        try:
+            return await task
+        finally:
+            if _local_probe_inflight.get("task") is task:
+                _local_probe_inflight["task"] = None
 
     @router.get("/ping")
     def ping_endpoints(request: Request):
@@ -1636,6 +1744,8 @@ def setup_model_routes(model_discovery):
         require_admin(request)
         db = SessionLocal()
         try:
+            if _disable_stale_cookbook_local_endpoints(db):
+                _invalidate_models_cache()
             rows = db.query(ModelEndpoint).order_by(ModelEndpoint.created_at).all()
             results = []
             for r in rows:
@@ -1643,67 +1753,11 @@ def setup_model_routes(model_discovery):
                 hidden = _hidden_model_ids(r)
                 pinned = _normalize_model_ids(getattr(r, "pinned_models", None))
                 visible = _visible_models(all_models, r.hidden_models, pinned)
-                # Endpoint counts as reachable if it has any model — including
-                # admin-pinned IDs that a probe would never surface.
-                status = "online" if (all_models or pinned) else "offline"
+                # Keep the list route cache-only. It feeds Settings →
+                # Added Models and must render immediately; explicit
+                # Refresh/Probe endpoints do the network work.
+                status = "online" if (all_models or pinned) else ("empty" if r.is_enabled else "offline")
                 ping = None
-                # When cached_models is empty, do a quick reachability probe.
-                # Bumped 1.0s → 3.5s because the user reported endpoints they
-                # were ACTIVELY chatting with showed "offline" — the previous
-                # 1s timeout was clipping live cloud endpoints (DeepSeek can
-                # take 1.5–2.5s on /v1/models when their region is under load,
-                # vLLM on a remote GPU box behind SSH can also push past 1s).
-                # 3.5s still keeps the picker render snappy in the common
-                # "everything's already cached" path because this branch only
-                # runs for endpoints with an empty cached_models.
-                if not all_models and not pinned and r.is_enabled:
-                    base_for_ping = _normalize_base(r.base_url)
-                    kind_for_ping = _effective_endpoint_kind(r, base_for_ping)
-                    ping_timeout = 10.0 if _classify_endpoint(base_for_ping, kind_for_ping) == "local" else 3.5
-                    ping = _ping_endpoint(r.base_url, r.api_key, timeout=ping_timeout)
-                    if ping.get("reachable"):
-                        status = "loading" if ping.get("loading") else "empty"
-                        if ping.get("loading"):
-                            base = _normalize_base(r.base_url)
-                            kind = _effective_endpoint_kind(r, base)
-                            results.append({
-                                "id": r.id,
-                                "name": r.name,
-                                "base_url": r.base_url,
-                                "has_key": bool(r.api_key),
-                                "api_key_fingerprint": _api_key_fingerprint(r.api_key),
-                                "is_enabled": r.is_enabled,
-                                "models": visible,
-                                "pinned_models": pinned,
-                                "hidden_count": len(hidden),
-                                "online": True,
-                                "status": status,
-                                "ping_error": (ping or {}).get("error") if ping else None,
-                                "model_type": getattr(r, "model_type", None) or "llm",
-                                "supports_tools": getattr(r, "supports_tools", None),
-                                "endpoint_kind": kind,
-                                "category": _classify_endpoint(base, kind),
-                                "model_refresh_mode": _endpoint_refresh_mode(r, kind),
-                                "model_refresh_interval": getattr(r, "model_refresh_interval", None),
-                                "model_refresh_timeout": getattr(r, "model_refresh_timeout", None),
-                            })
-                            continue
-                        # Best-effort: if the probe came back reachable, try
-                        # to populate cached_models in the background so the
-                        # NEXT picker load shows "online" instead of "empty".
-                        # Failure here is silent — we already returned the
-                        # "empty" status, and the existing background refresh
-                        # path will eventually fill it in too.
-                        try:
-                            probed = _probe_endpoint(r.base_url, r.api_key, timeout=max(5, int(ping_timeout)))
-                            if probed:
-                                r.cached_models = json.dumps(probed)
-                                db.commit()
-                                all_models = probed
-                                visible = _visible_models(all_models, r.hidden_models, pinned)
-                                status = "online"
-                        except Exception as _refill_err:
-                            logger.debug(f"opportunistic cached_models refill failed for {r.id}: {_refill_err!r}")
                 base = _normalize_base(r.base_url)
                 kind = _effective_endpoint_kind(r, base)
                 results.append({
@@ -1920,7 +1974,18 @@ def setup_model_routes(model_discovery):
                     ModelEndpoint.is_enabled == True  # noqa: E712
                 ).all()
             }
-            if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids):
+            current_default_id = settings.get("default_endpoint_id") or ""
+            current_default_ep = None
+            if current_default_id:
+                current_default_ep = db.query(ModelEndpoint).filter(
+                    ModelEndpoint.id == current_default_id
+                ).first()
+            if _default_endpoint_needs_assignment(
+                current_default_id,
+                enabled_ids,
+                current_default_endpoint=current_default_ep,
+                current_default_model=settings.get("default_model") or "",
+            ):
                 from src.endpoint_resolver import _first_chat_model
                 settings["default_endpoint_id"] = ep.id
                 settings["default_model"] = _first_chat_model(model_ids) or ""

routes/research/__init__.py

@@ -0,0 +1,5 @@
+"""Research route domain package (slice 2b, #4082/#4071).
+
+Contains research_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/research_routes.py re-exports from here.
+"""

routes/research/research_routes.py

@@ -0,0 +1,749 @@
+"""Research background task routes — /api/research/*."""
+
+import asyncio
+import json
+import logging
+import re
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import HTMLResponse, StreamingResponse
+from pydantic import BaseModel, Field
+from core.middleware import INTERNAL_TOOL_USER
+from src.endpoint_resolver import resolve_endpoint
+from src.auth_helpers import _auth_disabled, get_current_user
+from core.auth import RESERVED_USERNAMES
+from src.constants import DEEP_RESEARCH_DIR
+
+_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
+
+
+def _confine_research_path(session_id: str) -> Path:
+    """Return the resolved Path for session_id's JSON inside DEEP_RESEARCH_DIR.
+
+    Validates the session ID format and asserts containment after symlink
+    expansion. Raises HTTPException(400) on format failures, traversal
+    attempts, absolute-path injection, and symlink escape so every caller
+    gets a safe, confined path with no extra validation needed.
+    """
+    if not _SESSION_ID_RE.fullmatch(session_id):
+        raise HTTPException(400, "Invalid session ID")
+    root = Path(DEEP_RESEARCH_DIR).resolve()
+    candidate = (root / f"{session_id}.json").resolve()
+    try:
+        candidate.relative_to(root)
+    except ValueError:
+        raise HTTPException(400, "Invalid session ID")
+    return candidate
+
+
+logger = logging.getLogger(__name__)
+
+# Model-name substrings that are NOT chat/generation models — research must
+# never pick these as its model. An OpenAI-style endpoint often lists
+# `text-embedding-ada-002` etc. first in its model list, which is why research
+# was failing with "Cannot reach model 'text-embedding-ada-002'".
+_NON_CHAT_MODEL = (
+    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
+    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
+)
+
+_RESEARCH_IMAGE_BLOCKLIST = {
+    "cdn.shopify.com/s/files/1/0179/4388/7926/files/icon.png",
+}
+
+
+def _is_research_icon_or_logo_url(url: str) -> bool:
+    path = url.lower().split("?")[0]
+    return any(token in path for token in (
+        "/logo", "logo_", "-logo", "favicon", "apple-touch-icon",
+        "sprite", "icon-", "_icon", "/icons/", "badge",
+    ))
+
+
+def _research_thumbnail(data: dict) -> str:
+    """Pick the same first visible image the visual report uses as hero."""
+    hidden = set(data.get("hidden_images") or [])
+    seen = set()
+
+    def usable(image: str) -> bool:
+        image = str(image or "").strip()
+        if not image or image in seen or image in hidden:
+            return False
+        if not image.startswith("https://"):
+            return False
+        if image.endswith((".svg", ".ico", ".gif")):
+            return False
+        if any(blocked in image for blocked in _RESEARCH_IMAGE_BLOCKLIST):
+            return False
+        if _is_research_icon_or_logo_url(image):
+            return False
+        return True
+
+    for source in data.get("sources") or []:
+        if not isinstance(source, dict):
+            continue
+        image = str(source.get("image") or source.get("og_image") or "").strip()
+        if usable(image):
+            seen.add(image)
+            return image
+    for finding in data.get("raw_findings") or data.get("findings") or []:
+        if not isinstance(finding, dict):
+            continue
+        image = str(finding.get("image") or finding.get("og_image") or "").strip()
+        if usable(image):
+            seen.add(image)
+            return image
+    return ""
+
+
+def _first_chat_model(models) -> str:
+    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
+    for m in (models or []):
+        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
+            return m
+    return (models[0] if models else "")
+
+
+def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
+    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
+    owner = owner or getattr(sess, "owner", None) or None
+    url, model, headers = resolve_endpoint(
+        "research",
+        fallback_url=sess.endpoint_url,
+        fallback_model=sess.model,
+        fallback_headers=sess.headers,
+        owner=owner,
+    )
+    return url, model, headers
+
+
+def _owned_enabled_endpoint(db, owner, endpoint_id=None):
+    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
+    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
+    None if nothing visible matches.
+
+    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
+    owner = private, "the model picker only shows the endpoint to that user") and
+    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
+    api_key + base_url into research_handler.start_research(llm_endpoint=,
+    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
+    via the bare first-enabled fallback — would let a research-privileged user
+    spend ANOTHER user's API key/quota and reach whatever internal base_url they
+    configured. Mirrors webhook_routes._first_enabled_endpoint and
+    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
+    legacy mode).
+    """
+    from src.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if endpoint_id:
+        q = q.filter(ModelEndpoint.id == endpoint_id)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
+    """Resolve a ModelEndpoint row into (chat_url, model, headers).
+
+    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
+    panel-selected research endpoints. ChatGPT Subscription endpoints keep
+    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
+    """
+    from src.endpoint_resolver import (
+        build_chat_url,
+        build_headers,
+        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
+    )
+
+    try:
+        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
+    except Exception as e:
+        logger.warning("Could not resolve endpoint credentials for research: %s", e)
+        return None
+
+    ep_model = (model or "").strip()
+    if not ep_model:
+        try:
+            models = json.loads(ep.cached_models) if ep.cached_models else []
+            if models:
+                ep_model = _first_chat_model(models)
+        except Exception:
+            pass
+    if not ep_model:
+        return None
+    return build_chat_url(base), ep_model, build_headers(api_key, base)
+
+
+def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
+    router = APIRouter(tags=["research"])
+
+    def _require_user(request: Request) -> str:
+        """All research endpoints require an authenticated user. Research
+        data isn't owner-scoped in the on-disk JSON yet, so we at least
+        block anonymous access. Multi-tenant deploys should additionally
+        verify the session belongs to this user."""
+        user = get_current_user(request)
+        if not user:
+            if _auth_disabled():
+                return ""
+            raise HTTPException(401, "Not authenticated")
+        return user
+
+    def _validate_session_id(session_id: str) -> None:
+        if not _SESSION_ID_RE.fullmatch(session_id):
+            raise HTTPException(400, "Invalid session ID format")
+
+    def _owns_in_memory(session_id: str, user: str) -> bool:
+        """Ownership check for an in-flight (in-memory) research task.
+        Falls back to the on-disk JSON if the task has already finished."""
+        entry = research_handler._active_tasks.get(session_id)
+        if entry is not None:
+            return entry.get("owner", "") == user
+        # Task no longer in memory — check the persisted JSON.
+        try:
+            path = _confine_research_path(session_id)
+        except HTTPException:
+            return False
+        if not path.exists():
+            return False
+        try:
+            return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
+        except Exception:
+            return False
+
+    @router.get("/api/research/active")
+    async def research_active(request: Request):
+        """List all currently active (running) research tasks."""
+        user = _require_user(request)
+        active = []
+        for sid, entry in research_handler._active_tasks.items():
+            # SECURITY: only show this user's running tasks.
+            if entry.get("owner", "") != user:
+                continue
+            if entry.get("status") == "running":
+                active.append({
+                    "session_id": sid,
+                    "query": entry.get("query", ""),
+                    "status": "running",
+                    "progress": entry.get("progress", {}),
+                    "started_at": entry.get("started_at", 0),
+                })
+        return {"active": active}
+
+    @router.get("/api/research/status/{session_id}")
+    async def research_status(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        status = research_handler.get_status(session_id)
+        if status is None:
+            raise HTTPException(404, "No research found for this session")
+        return status
+
+    @router.post("/api/research/cancel/{session_id}")
+    async def research_cancel(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        cancelled = research_handler.cancel_research(session_id)
+        return {"cancelled": cancelled}
+
+    @router.post("/api/research/result/{session_id}")
+    async def research_result(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research result available")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        research_handler.clear_result(session_id)
+        return {"result": result, "sources": sources, "raw_findings": raw_findings}
+
+    def _assert_owns_research(session_id: str, user: str) -> None:
+        """404-not-403 ownership gate for a research session's on-disk JSON.
+        Use BEFORE returning any data or mutating the file."""
+        path = _confine_research_path(session_id)
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
+        except Exception:
+            raise HTTPException(404, "Research not found")
+        if owner != user:
+            raise HTTPException(404, "Research not found")
+
+    @router.get("/api/research/report/{session_id}")
+    async def research_report(session_id: str, request: Request):
+        """Serve the visual HTML report for a completed research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        logger.info(f"Visual report requested for session {session_id}")
+        try:
+            html_content = research_handler.get_report_html(session_id)
+        except Exception as e:
+            logger.error(f"Visual report generation error: {e}", exc_info=True)
+            raise HTTPException(500, f"Report generation failed: {e}")
+        if html_content is None:
+            logger.warning(f"No report data found for session {session_id}")
+            raise HTTPException(404, "No visual report available for this session")
+        return HTMLResponse(content=html_content)
+
+    class HideImageRequest(BaseModel):
+        url: str
+
+    @router.post("/api/research/{session_id}/hide-image")
+    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
+        """Mark an image URL as hidden for this research's visual report.
+        Persisted to the research JSON so subsequent /report renders skip it."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.hide_image(session_id, body.url)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.post("/api/research/{session_id}/unhide-images")
+    async def research_unhide_images(session_id: str, request: Request):
+        """Clear the hidden-images list for a research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.unhide_all_images(session_id)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.get("/api/research/library")
+    async def research_library(
+        request: Request,
+        search: Optional[str] = Query(None),
+        sort: str = Query("recent"),
+        limit: int = Query(50),
+        archived: bool = Query(False),
+    ):
+        user = _require_user(request)
+        """List all completed research for the Library panel."""
+        data_dir = Path(DEEP_RESEARCH_DIR)
+        items = []
+        for p in data_dir.glob("*.json"):
+            try:
+                d = json.loads(p.read_text(encoding="utf-8"))
+                # SECURITY: only show research belonging to this user. Legacy
+                # JSONs without an `owner` field are hidden — auth was the only
+                # gate before, so every user saw every other user's reports.
+                if d.get("owner") != user:
+                    continue
+                # Archived view shows ONLY archived reports; default hides them.
+                if bool(d.get("archived")) != archived:
+                    continue
+                query = d.get("query", "")
+                if search and search.lower() not in query.lower():
+                    continue
+                sources = d.get("sources", [])
+                items.append({
+                    "id": p.stem,
+                    "query": query,
+                    "category": d.get("category") or "",
+                    "source_count": len(sources),
+                    "status": d.get("status", "done"),
+                    "duration": d.get("stats", {}).get("Duration", ""),
+                    "rounds": d.get("stats", {}).get("Rounds", ""),
+                    "started_at": d.get("started_at", 0),
+                    "completed_at": d.get("completed_at", 0),
+                    "archived": bool(d.get("archived")),
+                    "thumbnail": _research_thumbnail(d),
+                })
+            except Exception:
+                continue
+
+        # Sort
+        if sort == "recent":
+            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
+        elif sort == "oldest":
+            items.sort(key=lambda x: x["completed_at"] or 0)
+        elif sort == "most-messages":
+            items.sort(key=lambda x: x["source_count"], reverse=True)
+        elif sort == "alpha":
+            items.sort(key=lambda x: x["query"].lower())
+
+        return {"research": items[:limit], "total": len(items)}
+
+    @router.get("/api/research/detail/{session_id}")
+    async def research_detail(session_id: str, request: Request):
+        """Return the full JSON for a single research result — sources,
+        summary, stats — used by the Library preview panel."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = _confine_research_path(session_id)
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as e:
+            raise HTTPException(500, f"Failed to read research: {e}")
+        # SECURITY: 404 (not 403) so we don't leak that the report exists.
+        if data.get("owner") != user:
+            raise HTTPException(404, "Research not found")
+        return data
+
+    @router.post("/api/research/{session_id}/archive")
+    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
+        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = _confine_research_path(session_id)
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("owner") != user:
+                raise HTTPException(404, "Research not found")
+            data["archived"] = bool(archived)
+            path.write_text(json.dumps(data), encoding="utf-8")
+        except HTTPException:
+            raise
+        except Exception as e:
+            raise HTTPException(500, f"Failed to update research: {e}")
+        return {"ok": True, "id": session_id, "archived": bool(archived)}
+
+    @router.delete("/api/research/{session_id}")
+    async def research_delete(session_id: str, request: Request):
+        """Delete a research result from disk."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        json_path = _confine_research_path(session_id)
+        deleted = False
+        if json_path.exists():
+            # SECURITY: verify ownership before letting the caller delete it.
+            try:
+                data = json.loads(json_path.read_text(encoding="utf-8"))
+                if data.get("owner") != user:
+                    raise HTTPException(404, "Research not found")
+            except HTTPException:
+                raise
+            except Exception:
+                raise HTTPException(404, "Research not found")
+            json_path.unlink()
+            deleted = True
+        return {"deleted": deleted}
+
+    # ------------------------------------------------------------------
+    # Panel endpoints — launch research without a chat session
+    # ------------------------------------------------------------------
+
+    class ResearchStartRequest(BaseModel):
+        query: str
+        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
+        max_rounds: int = Field(default=0, ge=0, le=20)
+        search_provider: Optional[str] = None
+        endpoint_id: Optional[str] = None
+        model: Optional[str] = None
+        max_time: int = Field(default=300, ge=60, le=1800)
+        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
+        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
+        category: Optional[str] = None
+
+    @router.post("/api/research/start")
+    async def research_start(body: ResearchStartRequest, request: Request):
+        """Launch a research job from the dedicated panel."""
+        from src.auth_helpers import require_privilege
+        user = require_privilege(request, "can_use_research")
+        if user == INTERNAL_TOOL_USER:
+            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
+            if tool_owner and tool_owner not in RESERVED_USERNAMES:
+                auth_mgr = getattr(request.app.state, "auth_manager", None)
+                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
+                    try:
+                        privs = auth_mgr.get_privileges(tool_owner) or {}
+                        if not privs.get("can_use_research", True):
+                            raise HTTPException(403, f"Your account is not allowed to can use research.")
+                    except HTTPException:
+                        raise
+                    except Exception:
+                        pass
+                user = tool_owner
+        session_id = f"rp-{uuid.uuid4().hex[:12]}"
+
+        if body.endpoint_id:
+            from src.database import SessionLocal
+            db = SessionLocal()
+            try:
+                # Owner-scoped: never resolve another user's private endpoint
+                # (and its decrypted api_key / internal base_url). A scoped miss
+                # reads as 404 so the endpoint's existence isn't revealed.
+                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
+                if not ep:
+                    raise HTTPException(404, "Endpoint not found or disabled")
+                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
+                if not resolved:
+                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
+                ep_url, ep_model, ep_headers = resolved
+            finally:
+                db.close()
+        else:
+            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
+            # When neither research nor utility is configured, use the user's
+            # configured DEFAULT model (default_endpoint_id/default_model) rather
+            # than arbitrarily grabbing the first enabled endpoint's first model
+            # (which surfaced gpt-3.5). "Default" should mean the default model.
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
+            if not ep_url:
+                from src.database import SessionLocal
+                db = SessionLocal()
+                try:
+                    # Owner-scoped first-enabled fallback: the caller's own rows
+                    # + legacy null-owner shared rows only — never borrow another
+                    # user's private endpoint/api_key. Same fix as the
+                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
+                    ep = _owned_enabled_endpoint(db, user)
+                    if ep:
+                        resolved = _resolve_endpoint_runtime(ep, owner=user)
+                        if resolved:
+                            ep_url, ep_model, ep_headers = resolved
+                finally:
+                    db.close()
+            if not ep_url:
+                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
+            if body.model:
+                ep_model = body.model
+
+        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
+        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
+        research_handler.start_research(
+            session_id=session_id,
+            query=body.query,
+            llm_endpoint=ep_url,
+            llm_model=ep_model,
+            max_time=body.max_time,
+            llm_headers=ep_headers,
+            max_rounds=effective_max_rounds,
+            search_provider=body.search_provider or None,
+            category=body.category or None,
+            extraction_timeout=body.extraction_timeout,
+            extraction_concurrency=body.extraction_concurrency,
+            owner=user,
+        )
+        return {"session_id": session_id, "status": "running", "query": body.query}
+
+    @router.get("/api/research/stream/{session_id}")
+    async def research_stream(session_id: str, request: Request):
+        """SSE stream of research progress events."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        async def _generate():
+            last_progress = None
+            while True:
+                status = research_handler.get_status(session_id)
+                if status is None:
+                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
+                    return
+                st = status.get("status", "")
+                progress = status.get("progress", {})
+                if progress != last_progress:
+                    last_progress = progress
+                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
+                if st != "running":
+                    final = {'status': st, 'final': True}
+                    task = research_handler._active_tasks.get(session_id, {})
+                    if st == "error" and task.get("result"):
+                        final['error'] = str(task["result"])[:500]
+                    yield f"data: {json.dumps(final)}\n\n"
+                    return
+                await asyncio.sleep(1.5)
+
+        return StreamingResponse(
+            _generate(),
+            media_type="text/event-stream",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
+    @router.post("/api/research/result-peek/{session_id}")
+    async def research_result_peek(session_id: str, request: Request):
+        """Get research result without clearing it (for panel use)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            p = _confine_research_path(session_id)
+            if p.exists():
+                d = json.loads(p.read_text(encoding="utf-8"))
+                return {
+                    "result": d.get("result", ""),
+                    "sources": d.get("sources", []),
+                    "raw_findings": d.get("raw_findings", []),
+                    "category": d.get("category") or "",
+                }
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
+
+    @router.post("/api/research/spinoff/{session_id}")
+    async def research_spinoff(session_id: str, request: Request):
+        """Create a new chat session pre-seeded with this research as context.
+
+        Reads the persisted research result + sources for `session_id`, creates
+        a fresh session (inheriting endpoint/model/headers from the source
+        session if available, otherwise from the resolved chat endpoint), and
+        injects a single system message containing the report and sources so
+        the user can ask follow-up questions in a clean conversation.
+        """
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        # SECURITY: gate on ownership before reading the persisted research —
+        # otherwise any authenticated user could spin off (and thereby read)
+        # another user's report by guessing its session ID. Mirrors every other
+        # endpoint in this file (see result_peek above).
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        if session_manager is None:
+            raise HTTPException(500, "session_manager not configured")
+
+        # Load research data — prefer in-memory result, fall back to disk
+        result = research_handler.get_result(session_id)
+        sources = research_handler.get_sources(session_id) or []
+        query = ""
+
+        path = _confine_research_path(session_id)
+        if path.exists():
+            try:
+                disk = json.loads(path.read_text(encoding="utf-8"))
+                if not result:
+                    result = disk.get("result")
+                if not sources:
+                    sources = disk.get("sources", []) or []
+                query = disk.get("query", "") or ""
+            except Exception as e:
+                logger.warning(f"Could not read research JSON for spinoff: {e}")
+
+        if not result:
+            raise HTTPException(404, "No research result available for this session")
+
+        # Inherit endpoint/model/headers from the source session when possible.
+        # For panel-launched research (rp-* IDs), there is no chat session, so
+        # fall back through the same chain as /api/research/start: research →
+        # utility → first enabled endpoint in the DB.
+        ep_url, ep_model, ep_headers = "", "", {}
+        try:
+            src_sess = session_manager.get_session(session_id)
+            ep_url = src_sess.endpoint_url or ""
+            ep_model = src_sess.model or ""
+            ep_headers = dict(src_sess.headers or {})
+        except KeyError:
+            pass
+
+        def _merge(r_url, r_model, r_headers):
+            nonlocal ep_url, ep_model, ep_headers
+            if not ep_url and r_url:
+                ep_url = r_url
+            if not ep_model and r_model:
+                ep_model = r_model
+            if not ep_headers and r_headers:
+                ep_headers = dict(r_headers)
+
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("chat", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("research", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("utility", owner=user))
+        if not ep_url or not ep_model:
+            # Last resort: this user's enabled endpoint, plus legacy shared rows.
+            from src.database import SessionLocal
+            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
+            db = SessionLocal()
+            try:
+                ep = _owned_enabled_endpoint(db, user)
+                if ep:
+                    base = normalize_base(ep.base_url)
+                    fallback_url = build_chat_url(base)
+                    fallback_headers = build_headers(ep.api_key, base)
+                    fallback_model = ""
+                    if ep.cached_models:
+                        try:
+                            models = json.loads(ep.cached_models)
+                            if models:
+                                fallback_model = _first_chat_model(models)
+                        except Exception:
+                            pass
+                    _merge(fallback_url, fallback_model, fallback_headers)
+            finally:
+                db.close()
+
+        if not ep_url or not ep_model:
+            raise HTTPException(400, "No endpoint configured — add one in Settings first")
+
+        # Create new session
+        new_sid = str(uuid.uuid4())
+
+        title_query = (query or "research").strip()
+        if len(title_query) > 60:
+            title_query = title_query[:57] + "…"
+        new_name = f"Follow-up: {title_query}"
+
+        new_sess = session_manager.create_session(
+            session_id=new_sid,
+            name=new_name,
+            endpoint_url=ep_url,
+            model=ep_model,
+            rag=False,
+            owner=user,
+        )
+        if ep_headers:
+            new_sess.headers = ep_headers
+            session_manager.save_sessions()
+        try:
+            from src.event_bus import fire_event
+            fire_event("session_created", user)
+        except Exception:
+            logger.debug("session_created event dispatch failed", exc_info=True)
+
+        # Build the priming system message — report only, no sources injected.
+        # The user can open the visual report for source details; keeping sources
+        # out of the chat context saves tokens and avoids the AI fabricating
+        # citations.
+        date_str = datetime.utcnow().strftime("%Y-%m-%d")
+        primer = (
+            f"[Research context — {date_str}]\n\n"
+            f"The user previously ran a deep research investigation. Use the "
+            f"report below as your primary knowledge base when answering "
+            f"follow-up questions. If the user asks something not covered, "
+            f"say so plainly rather than guessing.\n\n"
+            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
+            f"=== REPORT ===\n{result}"
+        )
+
+        from core.models import ChatMessage
+        new_sess.add_message(ChatMessage(
+            role="system",
+            content=primer,
+            metadata={"research_spinoff_from": session_id},
+        ))
+        session_manager.save_sessions()
+
+        return {
+            "session_id": new_sid,
+            "name": new_name,
+            "source_count": len(sources),
+        }
+
+    return router

routes/research_routes.py

@@ -1,678 +1,17 @@
-"""Research background task routes — /api/research/*."""
+"""Backward-compat shim — canonical location is routes/research/research_routes.py.
 
-import asyncio
-import json
-import logging
-import re
-import uuid
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.research_routes``, ``from routes.research_routes import X``,
+``importlib.import_module("routes.research_routes")``, and
+``monkeypatch.setattr("routes.research_routes.ATTR", ...)`` (string-targeted
+patch used by ``test_research_owner_scope_routes.py``) all operate on the
+*same* object the application actually uses. Keeps existing import paths
+working after slice 2b (#4082/#4071). Source-introspection tests read the
+canonical file by path.
+"""
 
-from fastapi import APIRouter, HTTPException, Query, Request
-from fastapi.responses import HTMLResponse, StreamingResponse
-from pydantic import BaseModel, Field
-from core.middleware import INTERNAL_TOOL_USER
-from src.endpoint_resolver import resolve_endpoint
-from src.auth_helpers import _auth_disabled, get_current_user
-from core.auth import RESERVED_USERNAMES
-from src.constants import DEEP_RESEARCH_DIR
+import sys as _sys
 
-_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
+from routes.research import research_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-# Model-name substrings that are NOT chat/generation models — research must
-# never pick these as its model. An OpenAI-style endpoint often lists
-# `text-embedding-ada-002` etc. first in its model list, which is why research
-# was failing with "Cannot reach model 'text-embedding-ada-002'".
-_NON_CHAT_MODEL = (
-    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
-    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
-)
-
-
-def _first_chat_model(models) -> str:
-    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
-    for m in (models or []):
-        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
-            return m
-    return (models[0] if models else "")
-
-
-def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
-    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
-    owner = owner or getattr(sess, "owner", None) or None
-    url, model, headers = resolve_endpoint(
-        "research",
-        fallback_url=sess.endpoint_url,
-        fallback_model=sess.model,
-        fallback_headers=sess.headers,
-        owner=owner,
-    )
-    return url, model, headers
-
-
-def _owned_enabled_endpoint(db, owner, endpoint_id=None):
-    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
-    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
-    None if nothing visible matches.
-
-    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
-    owner = private, "the model picker only shows the endpoint to that user") and
-    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
-    api_key + base_url into research_handler.start_research(llm_endpoint=,
-    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
-    via the bare first-enabled fallback — would let a research-privileged user
-    spend ANOTHER user's API key/quota and reach whatever internal base_url they
-    configured. Mirrors webhook_routes._first_enabled_endpoint and
-    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
-    legacy mode).
-    """
-    from src.database import ModelEndpoint
-    from src.auth_helpers import owner_filter
-    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
-    if endpoint_id:
-        q = q.filter(ModelEndpoint.id == endpoint_id)
-    return owner_filter(q, ModelEndpoint, owner).first()
-
-
-def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
-    """Resolve a ModelEndpoint row into (chat_url, model, headers).
-
-    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
-    panel-selected research endpoints. ChatGPT Subscription endpoints keep
-    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
-    """
-    from src.endpoint_resolver import (
-        build_chat_url,
-        build_headers,
-        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
-    )
-
-    try:
-        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
-    except Exception as e:
-        logger.warning("Could not resolve endpoint credentials for research: %s", e)
-        return None
-
-    ep_model = (model or "").strip()
-    if not ep_model:
-        try:
-            models = json.loads(ep.cached_models) if ep.cached_models else []
-            if models:
-                ep_model = _first_chat_model(models)
-        except Exception:
-            pass
-    if not ep_model:
-        return None
-    return build_chat_url(base), ep_model, build_headers(api_key, base)
-
-
-def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
-    router = APIRouter(tags=["research"])
-
-    def _require_user(request: Request) -> str:
-        """All research endpoints require an authenticated user. Research
-        data isn't owner-scoped in the on-disk JSON yet, so we at least
-        block anonymous access. Multi-tenant deploys should additionally
-        verify the session belongs to this user."""
-        user = get_current_user(request)
-        if not user:
-            if _auth_disabled():
-                return ""
-            raise HTTPException(401, "Not authenticated")
-        return user
-
-    def _validate_session_id(session_id: str) -> None:
-        if not _SESSION_ID_RE.fullmatch(session_id):
-            raise HTTPException(400, "Invalid session ID format")
-
-    def _owns_in_memory(session_id: str, user: str) -> bool:
-        """Ownership check for an in-flight (in-memory) research task.
-        Falls back to the on-disk JSON if the task has already finished."""
-        entry = research_handler._active_tasks.get(session_id)
-        if entry is not None:
-            return entry.get("owner", "") == user
-        # Task no longer in memory — check the persisted JSON.
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            return False
-        try:
-            return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
-        except Exception:
-            return False
-
-    @router.get("/api/research/active")
-    async def research_active(request: Request):
-        """List all currently active (running) research tasks."""
-        user = _require_user(request)
-        active = []
-        for sid, entry in research_handler._active_tasks.items():
-            # SECURITY: only show this user's running tasks.
-            if entry.get("owner", "") != user:
-                continue
-            if entry.get("status") == "running":
-                active.append({
-                    "session_id": sid,
-                    "query": entry.get("query", ""),
-                    "status": "running",
-                    "progress": entry.get("progress", {}),
-                    "started_at": entry.get("started_at", 0),
-                })
-        return {"active": active}
-
-    @router.get("/api/research/status/{session_id}")
-    async def research_status(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        status = research_handler.get_status(session_id)
-        if status is None:
-            raise HTTPException(404, "No research found for this session")
-        return status
-
-    @router.post("/api/research/cancel/{session_id}")
-    async def research_cancel(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        cancelled = research_handler.cancel_research(session_id)
-        return {"cancelled": cancelled}
-
-    @router.post("/api/research/result/{session_id}")
-    async def research_result(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research result available")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
-
-    def _assert_owns_research(session_id: str, user: str) -> None:
-        """404-not-403 ownership gate for a research session's on-disk JSON.
-        Use BEFORE returning any data or mutating the file."""
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
-        except Exception:
-            raise HTTPException(404, "Research not found")
-        if owner != user:
-            raise HTTPException(404, "Research not found")
-
-    @router.get("/api/research/report/{session_id}")
-    async def research_report(session_id: str, request: Request):
-        """Serve the visual HTML report for a completed research session."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        logger.info(f"Visual report requested for session {session_id}")
-        try:
-            html_content = research_handler.get_report_html(session_id)
-        except Exception as e:
-            logger.error(f"Visual report generation error: {e}", exc_info=True)
-            raise HTTPException(500, f"Report generation failed: {e}")
-        if html_content is None:
-            logger.warning(f"No report data found for session {session_id}")
-            raise HTTPException(404, "No visual report available for this session")
-        return HTMLResponse(content=html_content)
-
-    class HideImageRequest(BaseModel):
-        url: str
-
-    @router.post("/api/research/{session_id}/hide-image")
-    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
-        """Mark an image URL as hidden for this research's visual report.
-        Persisted to the research JSON so subsequent /report renders skip it."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.hide_image(session_id, body.url)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.post("/api/research/{session_id}/unhide-images")
-    async def research_unhide_images(session_id: str, request: Request):
-        """Clear the hidden-images list for a research session."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.unhide_all_images(session_id)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.get("/api/research/library")
-    async def research_library(
-        request: Request,
-        search: Optional[str] = Query(None),
-        sort: str = Query("recent"),
-        limit: int = Query(50),
-        archived: bool = Query(False),
-    ):
-        user = _require_user(request)
-        """List all completed research for the Library panel."""
-        data_dir = Path(DEEP_RESEARCH_DIR)
-        items = []
-        for p in data_dir.glob("*.json"):
-            try:
-                d = json.loads(p.read_text(encoding="utf-8"))
-                # SECURITY: only show research belonging to this user. Legacy
-                # JSONs without an `owner` field are hidden — auth was the only
-                # gate before, so every user saw every other user's reports.
-                if d.get("owner") != user:
-                    continue
-                # Archived view shows ONLY archived reports; default hides them.
-                if bool(d.get("archived")) != archived:
-                    continue
-                query = d.get("query", "")
-                if search and search.lower() not in query.lower():
-                    continue
-                sources = d.get("sources", [])
-                items.append({
-                    "id": p.stem,
-                    "query": query,
-                    "category": d.get("category") or "",
-                    "source_count": len(sources),
-                    "status": d.get("status", "done"),
-                    "duration": d.get("stats", {}).get("Duration", ""),
-                    "rounds": d.get("stats", {}).get("Rounds", ""),
-                    "started_at": d.get("started_at", 0),
-                    "completed_at": d.get("completed_at", 0),
-                    "archived": bool(d.get("archived")),
-                })
-            except Exception:
-                continue
-
-        # Sort
-        if sort == "recent":
-            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
-        elif sort == "oldest":
-            items.sort(key=lambda x: x["completed_at"] or 0)
-        elif sort == "most-messages":
-            items.sort(key=lambda x: x["source_count"], reverse=True)
-        elif sort == "alpha":
-            items.sort(key=lambda x: x["query"].lower())
-
-        return {"research": items[:limit], "total": len(items)}
-
-    @router.get("/api/research/detail/{session_id}")
-    async def research_detail(session_id: str, request: Request):
-        """Return the full JSON for a single research result — sources,
-        summary, stats — used by the Library preview panel."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-        except Exception as e:
-            raise HTTPException(500, f"Failed to read research: {e}")
-        # SECURITY: 404 (not 403) so we don't leak that the report exists.
-        if data.get("owner") != user:
-            raise HTTPException(404, "Research not found")
-        return data
-
-    @router.post("/api/research/{session_id}/archive")
-    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
-        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-            if data.get("owner") != user:
-                raise HTTPException(404, "Research not found")
-            data["archived"] = bool(archived)
-            path.write_text(json.dumps(data), encoding="utf-8")
-        except HTTPException:
-            raise
-        except Exception as e:
-            raise HTTPException(500, f"Failed to update research: {e}")
-        return {"ok": True, "id": session_id, "archived": bool(archived)}
-
-    @router.delete("/api/research/{session_id}")
-    async def research_delete(session_id: str, request: Request):
-        """Delete a research result from disk."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        data_dir = Path(DEEP_RESEARCH_DIR)
-        json_path = data_dir / f"{session_id}.json"
-        deleted = False
-        if json_path.exists():
-            # SECURITY: verify ownership before letting the caller delete it.
-            try:
-                data = json.loads(json_path.read_text(encoding="utf-8"))
-                if data.get("owner") != user:
-                    raise HTTPException(404, "Research not found")
-            except HTTPException:
-                raise
-            except Exception:
-                raise HTTPException(404, "Research not found")
-            json_path.unlink()
-            deleted = True
-        return {"deleted": deleted}
-
-    # ------------------------------------------------------------------
-    # Panel endpoints — launch research without a chat session
-    # ------------------------------------------------------------------
-
-    class ResearchStartRequest(BaseModel):
-        query: str
-        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
-        max_rounds: int = Field(default=0, ge=0, le=20)
-        search_provider: Optional[str] = None
-        endpoint_id: Optional[str] = None
-        model: Optional[str] = None
-        max_time: int = Field(default=300, ge=60, le=1800)
-        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
-        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
-        category: Optional[str] = None
-
-    @router.post("/api/research/start")
-    async def research_start(body: ResearchStartRequest, request: Request):
-        """Launch a research job from the dedicated panel."""
-        from src.auth_helpers import require_privilege
-        user = require_privilege(request, "can_use_research")
-        if user == INTERNAL_TOOL_USER:
-            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
-            if tool_owner and tool_owner not in RESERVED_USERNAMES:
-                auth_mgr = getattr(request.app.state, "auth_manager", None)
-                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
-                    try:
-                        privs = auth_mgr.get_privileges(tool_owner) or {}
-                        if not privs.get("can_use_research", True):
-                            raise HTTPException(403, f"Your account is not allowed to can use research.")
-                    except HTTPException:
-                        raise
-                    except Exception:
-                        pass
-                user = tool_owner
-        session_id = f"rp-{uuid.uuid4().hex[:12]}"
-
-        if body.endpoint_id:
-            from src.database import SessionLocal
-            db = SessionLocal()
-            try:
-                # Owner-scoped: never resolve another user's private endpoint
-                # (and its decrypted api_key / internal base_url). A scoped miss
-                # reads as 404 so the endpoint's existence isn't revealed.
-                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
-                if not ep:
-                    raise HTTPException(404, "Endpoint not found or disabled")
-                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
-                if not resolved:
-                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
-                ep_url, ep_model, ep_headers = resolved
-            finally:
-                db.close()
-        else:
-            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
-            # When neither research nor utility is configured, use the user's
-            # configured DEFAULT model (default_endpoint_id/default_model) rather
-            # than arbitrarily grabbing the first enabled endpoint's first model
-            # (which surfaced gpt-3.5). "Default" should mean the default model.
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
-            if not ep_url:
-                from src.database import SessionLocal
-                db = SessionLocal()
-                try:
-                    # Owner-scoped first-enabled fallback: the caller's own rows
-                    # + legacy null-owner shared rows only — never borrow another
-                    # user's private endpoint/api_key. Same fix as the
-                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
-                    ep = _owned_enabled_endpoint(db, user)
-                    if ep:
-                        resolved = _resolve_endpoint_runtime(ep, owner=user)
-                        if resolved:
-                            ep_url, ep_model, ep_headers = resolved
-                finally:
-                    db.close()
-            if not ep_url:
-                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
-            if body.model:
-                ep_model = body.model
-
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
-        research_handler.start_research(
-            session_id=session_id,
-            query=body.query,
-            llm_endpoint=ep_url,
-            llm_model=ep_model,
-            max_time=body.max_time,
-            llm_headers=ep_headers,
-            max_rounds=effective_max_rounds,
-            search_provider=body.search_provider or None,
-            category=body.category or None,
-            extraction_timeout=body.extraction_timeout,
-            extraction_concurrency=body.extraction_concurrency,
-            owner=user,
-        )
-        return {"session_id": session_id, "status": "running", "query": body.query}
-
-    @router.get("/api/research/stream/{session_id}")
-    async def research_stream(session_id: str, request: Request):
-        """SSE stream of research progress events."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        async def _generate():
-            last_progress = None
-            while True:
-                status = research_handler.get_status(session_id)
-                if status is None:
-                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
-                    return
-                st = status.get("status", "")
-                progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
-                if st != "running":
-                    final = {'status': st, 'final': True}
-                    task = research_handler._active_tasks.get(session_id, {})
-                    if st == "error" and task.get("result"):
-                        final['error'] = str(task["result"])[:500]
-                    yield f"data: {json.dumps(final)}\n\n"
-                    return
-                await asyncio.sleep(1.5)
-
-        return StreamingResponse(
-            _generate(),
-            media_type="text/event-stream",
-            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
-        )
-
-    @router.post("/api/research/result-peek/{session_id}")
-    async def research_result_peek(session_id: str, request: Request):
-        """Get research result without clearing it (for panel use)."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-            if p.exists():
-                d = json.loads(p.read_text(encoding="utf-8"))
-                return {
-                    "result": d.get("result", ""),
-                    "sources": d.get("sources", []),
-                    "raw_findings": d.get("raw_findings", []),
-                    "category": d.get("category") or "",
-                }
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
-
-    @router.post("/api/research/spinoff/{session_id}")
-    async def research_spinoff(session_id: str, request: Request):
-        """Create a new chat session pre-seeded with this research as context.
-
-        Reads the persisted research result + sources for `session_id`, creates
-        a fresh session (inheriting endpoint/model/headers from the source
-        session if available, otherwise from the resolved chat endpoint), and
-        injects a single system message containing the report and sources so
-        the user can ask follow-up questions in a clean conversation.
-        """
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        # SECURITY: gate on ownership before reading the persisted research —
-        # otherwise any authenticated user could spin off (and thereby read)
-        # another user's report by guessing its session ID. Mirrors every other
-        # endpoint in this file (see result_peek above).
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        if session_manager is None:
-            raise HTTPException(500, "session_manager not configured")
-
-        # Load research data — prefer in-memory result, fall back to disk
-        result = research_handler.get_result(session_id)
-        sources = research_handler.get_sources(session_id) or []
-        query = ""
-
-        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
-        if path.exists():
-            try:
-                disk = json.loads(path.read_text(encoding="utf-8"))
-                if not result:
-                    result = disk.get("result")
-                if not sources:
-                    sources = disk.get("sources", []) or []
-                query = disk.get("query", "") or ""
-            except Exception as e:
-                logger.warning(f"Could not read research JSON for spinoff: {e}")
-
-        if not result:
-            raise HTTPException(404, "No research result available for this session")
-
-        # Inherit endpoint/model/headers from the source session when possible.
-        # For panel-launched research (rp-* IDs), there is no chat session, so
-        # fall back through the same chain as /api/research/start: research →
-        # utility → first enabled endpoint in the DB.
-        ep_url, ep_model, ep_headers = "", "", {}
-        try:
-            src_sess = session_manager.get_session(session_id)
-            ep_url = src_sess.endpoint_url or ""
-            ep_model = src_sess.model or ""
-            ep_headers = dict(src_sess.headers or {})
-        except KeyError:
-            pass
-
-        def _merge(r_url, r_model, r_headers):
-            nonlocal ep_url, ep_model, ep_headers
-            if not ep_url and r_url:
-                ep_url = r_url
-            if not ep_model and r_model:
-                ep_model = r_model
-            if not ep_headers and r_headers:
-                ep_headers = dict(r_headers)
-
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("chat", owner=user))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("research", owner=user))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("utility", owner=user))
-        if not ep_url or not ep_model:
-            # Last resort: this user's enabled endpoint, plus legacy shared rows.
-            from src.database import SessionLocal
-            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
-            db = SessionLocal()
-            try:
-                ep = _owned_enabled_endpoint(db, user)
-                if ep:
-                    base = normalize_base(ep.base_url)
-                    fallback_url = build_chat_url(base)
-                    fallback_headers = build_headers(ep.api_key, base)
-                    fallback_model = ""
-                    if ep.cached_models:
-                        try:
-                            models = json.loads(ep.cached_models)
-                            if models:
-                                fallback_model = _first_chat_model(models)
-                        except Exception:
-                            pass
-                    _merge(fallback_url, fallback_model, fallback_headers)
-            finally:
-                db.close()
-
-        if not ep_url or not ep_model:
-            raise HTTPException(400, "No endpoint configured — add one in Settings first")
-
-        # Create new session
-        new_sid = str(uuid.uuid4())
-
-        title_query = (query or "research").strip()
-        if len(title_query) > 60:
-            title_query = title_query[:57] + "…"
-        new_name = f"Follow-up: {title_query}"
-
-        new_sess = session_manager.create_session(
-            session_id=new_sid,
-            name=new_name,
-            endpoint_url=ep_url,
-            model=ep_model,
-            rag=False,
-            owner=user,
-        )
-        if ep_headers:
-            new_sess.headers = ep_headers
-            session_manager.save_sessions()
-        try:
-            from src.event_bus import fire_event
-            fire_event("session_created", user)
-        except Exception:
-            logger.debug("session_created event dispatch failed", exc_info=True)
-
-        # Build the priming system message — report only, no sources injected.
-        # The user can open the visual report for source details; keeping sources
-        # out of the chat context saves tokens and avoids the AI fabricating
-        # citations.
-        date_str = datetime.utcnow().strftime("%Y-%m-%d")
-        primer = (
-            f"[Research context — {date_str}]\n\n"
-            f"The user previously ran a deep research investigation. Use the "
-            f"report below as your primary knowledge base when answering "
-            f"follow-up questions. If the user asks something not covered, "
-            f"say so plainly rather than guessing.\n\n"
-            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
-            f"=== REPORT ===\n{result}"
-        )
-
-        from core.models import ChatMessage
-        new_sess.add_message(ChatMessage(
-            role="system",
-            content=primer,
-            metadata={"research_spinoff_from": session_id},
-        ))
-        session_manager.save_sessions()
-
-        return {
-            "session_id": new_sid,
-            "name": new_name,
-            "source_count": len(sources),
-        }
-
-    return router
+_sys.modules[__name__] = _canonical

routes/session_routes.py

@@ -162,7 +162,7 @@ def _persist_session_headers(session_id: str, headers: dict | None) -> None:
         db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
         if db_session:
             db_session.headers = headers or {}
-            db_session.updated_at = datetime.utcnow()
+            db_session.updated_at = utcnow_naive()
             db.commit()
     except Exception:
         db.rollback()
@@ -223,8 +223,8 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         # purge exists only to catch ghosts the frontend missed (tab close,
         # crash). Only clean up rows old enough to be definitely orphaned.
         try:
-            from datetime import datetime as _dt, timedelta as _td
-            _cutoff = _dt.utcnow() - _td(minutes=10)
+            from datetime import timedelta as _td
+            _cutoff = utcnow_naive() - _td(minutes=10)
             _purge_db = SessionLocal()
             try:
                 from core.database import ChatMessage as _DbMsg
@@ -470,7 +470,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == sid).first()
                 if db_session:
                     db_session.folder = folder if folder else None
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
                     result["folder"] = folder if folder else None
             finally:
@@ -517,7 +517,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                     db_session.model = model
                     db_session.endpoint_url = endpoint_url
                     db_session.headers = session.headers or {}
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
             finally:
                 db.close()
@@ -646,7 +646,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == sid).first()
                 if db_session:
                     db_session.archived = True
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
                     
                     # Update in memory if it exists
@@ -680,7 +680,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             if not db_session:
                 raise HTTPException(404, f"Session {sid} not found")
             db_session.archived = False
-            db_session.updated_at = datetime.utcnow()
+            db_session.updated_at = utcnow_naive()
             db.commit()
             # Reload into session manager so it appears in the active list
             try:
@@ -890,7 +890,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
                 if db_session:
                     db_session.is_important = important
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     db.commit()
 
                     # Update in memory if it exists
@@ -979,7 +979,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             metadata={
                 "compacted": True,
                 "summarized_count": len(older),
-                "timestamp": datetime.utcnow().isoformat(),
+                "timestamp": utcnow_naive().isoformat(),
             },
         )
         new_history = [summary_msg] + recent
@@ -1256,7 +1256,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db_session_q.first()
                 if db_session:
                     db_session.folder = folder_name
-                    db_session.updated_at = datetime.utcnow()
+                    db_session.updated_at = utcnow_naive()
                     updated += 1
             db.commit()
         except Exception as e:

routes/shell_routes.py

@@ -16,6 +16,11 @@ from pathlib import Path
 from typing import Dict, Any
 from core.platform_compat import IS_APPLE_SILICON, which_tool
 from core.middleware import INTERNAL_TOOL_USER
+from src.host_docker_access import (
+    HOST_DOCKER_ACCESS_HINT,
+    host_docker_access_enabled as _host_docker_access_enabled,
+    running_in_container as _running_in_container,
+)
 from src.optional_deps import prepare_optional_dependency_import
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
@@ -103,32 +108,17 @@ logger = logging.getLogger(__name__)
 PTY_SUPPORTED = pty is not None and fcntl is not None and hasattr(os, "setsid")
 
 
-DOCKER_IN_CONTAINER_HINT = (
-    "Not available inside the Odysseus container by design. The image ships no "
-    "docker CLI and no host socket is mounted. Run Docker-backed launches on a "
-    "remote server, where docker is checked over SSH. Mounting /var/run/docker.sock "
-    "into the container would grant it host-root access, so only do that if you "
-    "accept that risk."
-)
-
-
-def _running_in_container(dockerenv_path="/.dockerenv", cgroup_path="/proc/1/cgroup"):
-    if os.path.exists(dockerenv_path):
-        return True
-    try:
-        with open(cgroup_path, "r", encoding="utf-8") as fh:
-            contents = fh.read()
-    except OSError:
-        return False
-    return any(token in contents for token in ("docker", "containerd", "kubepods"))
+DOCKER_IN_CONTAINER_HINT = HOST_DOCKER_ACCESS_HINT
 
 
 DockerRowStatus = namedtuple("DockerRowStatus", ["applicable", "install_hint"])
 PackageUpdateStatus = namedtuple("PackageUpdateStatus", ["available", "note"])
 
 
-def _docker_row_status(*, on_remote, in_container, installed, default_hint):
-    local_docker_unavailable = not on_remote and in_container and not installed
+def _docker_row_status(
+    *, on_remote, in_container, installed, default_hint, host_docker_access=False
+):
+    local_docker_unavailable = not on_remote and in_container and not host_docker_access
     if local_docker_unavailable:
         return DockerRowStatus(applicable=False, install_hint=DOCKER_IN_CONTAINER_HINT)
     return DockerRowStatus(applicable=True, install_hint=default_hint)
@@ -1159,7 +1149,7 @@ def setup_shell_routes() -> APIRouter:
             {
                 "name": "diffusers",
                 "pip": "diffusers[torch]",
-                "desc": "Image generation pipelines (SD, Flux) with PyTorch",
+                "desc": "Image generation/editing pipelines (SD, Flux) with PyTorch",
                 "category": "Image",
                 "target": "remote",
             },
@@ -1510,6 +1500,9 @@ def setup_shell_routes() -> APIRouter:
                     in_container=_running_in_container() if not on_remote else False,
                     installed=pkg["installed"],
                     default_hint=pkg.get("install_hint"),
+                    host_docker_access=(
+                        _host_docker_access_enabled() if not on_remote else False
+                    ),
                 )
                 pkg["applicable"] = status.applicable
                 pkg["install_hint"] = status.install_hint

routes/task_routes.py

@@ -594,6 +594,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         cache_tables = {
             "summarize_emails": ("email_summaries",),
             "draft_email_replies": ("email_ai_replies",),
+            "email_auto_translate": ("email_translations",),
             "extract_email_events": ("email_calendar_extractions",),
             "learn_sender_signatures": ("sender_signatures",),
             "check_email_urgency": ("email_tags", "email_urgency_alerts"),
@@ -893,10 +894,11 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         return {"ok": True, "message": "Task stopped"}
 
     @router.get("/runs/recent")
-    async def list_recent_runs(request: Request, limit: int = 50):
+    async def list_recent_runs(request: Request, limit: int = 50, max_result_chars: int = 6000):
         """Recent task runs across ALL tasks for this owner. Drives the Activity view."""
         user = _owner(request)
         limit = max(1, min(limit, 200))
+        max_result_chars = max(500, min(max_result_chars, 20000))
         db = SessionLocal()
         try:
             q = db.query(TaskRun, ScheduledTask).join(
@@ -930,10 +932,20 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 deduped.append((r, t))
                 if len(deduped) >= limit:
                     break
+
+            def _clip_run(r: TaskRun) -> dict:
+                d = _run_to_dict(r)
+                for key in ("result", "error"):
+                    val = d.get(key)
+                    if isinstance(val, str) and len(val) > max_result_chars:
+                        d[key] = val[:max_result_chars].rstrip() + "\n\n[Activity preview truncated]"
+                return d
+
             return {
+                "has_more": len(rows) > len(deduped),
                 "runs": [
                     {
-                        **_run_to_dict(r),
+                        **_clip_run(r),
                         "task_name": _display_task_name(t),
                         "task_type": t.task_type or "llm",
                         "action": t.action,

routes/upload_routes.py

@@ -6,11 +6,11 @@ import asyncio
 import shutil
 import uuid
 from pathlib import Path
-from fastapi import APIRouter, Request, File, UploadFile, HTTPException
-from typing import List
+from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
+from typing import List, Optional
 import logging
 from core.middleware import require_admin
-from core.database import SessionLocal, GalleryImage
+from core.database import SessionLocal, GalleryImage, Session as DbSession
 from src.auth_helpers import effective_user
 from src.constants import GENERATED_IMAGES_DIR
 from src.upload_handler import count_recent_uploads
@@ -56,7 +56,17 @@ def setup_upload_routes(upload_handler):
 
         raise HTTPException(404, "File not found")
 
-    def _promote_chat_image_to_gallery(meta: dict, owner: str | None) -> str | None:
+    def _valid_session_id_for_owner(db, session_id: str | None, owner: str | None) -> str | None:
+        if not session_id:
+            return None
+        sess = db.query(DbSession).filter(DbSession.id == session_id).first()
+        if not sess:
+            return None
+        if owner and sess.owner and sess.owner != owner:
+            return None
+        return session_id
+
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
         """Make chat-uploaded images visible in Gallery without changing chat storage."""
         is_image_file = getattr(upload_handler, "is_image_file", None)
         if not callable(is_image_file):
@@ -105,6 +115,7 @@ def setup_upload_routes(upload_handler):
                 prompt=meta.get("name") or "Chat upload",
                 model="chat-upload",
                 owner=owner,
+                session_id=_valid_session_id_for_owner(db, session_id, owner),
                 file_hash=file_hash,
                 width=meta.get("width"),
                 height=meta.get("height"),
@@ -120,8 +131,14 @@ def setup_upload_routes(upload_handler):
             db.close()
     
     @router.post("")
-    async def api_upload(request: Request, files: List[UploadFile] = File(...)):
+    async def api_upload(
+        request: Request,
+        files: List[UploadFile] = File(...),
+        session_id: Optional[str] = Form(None),
+    ):
         """Upload files with enhanced security and organization."""
+        if not isinstance(session_id, str):
+            session_id = None
         if not files:
             raise HTTPException(400, "No files uploaded")
             
@@ -148,7 +165,7 @@ def setup_upload_routes(upload_handler):
             try:
                 owner = effective_user(request)
                 meta = upload_handler.save_upload(u, client_ip, owner=owner)
-                gallery_id = _promote_chat_image_to_gallery(meta, owner)
+                gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
                 item = {
                     "id": meta["id"],
                     "name": meta["name"],
@@ -263,6 +280,32 @@ def setup_upload_routes(upload_handler):
         os.makedirs(cache_dir, exist_ok=True)
         return os.path.join(cache_dir, file_id + ".txt")
 
+    def _sync_gallery_caption_for_upload(info: dict | None, owner: str | None, text: str) -> None:
+        """Copy upload OCR/vision text onto the promoted gallery image row."""
+        if not info:
+            return
+        file_hash = info.get("hash")
+        if not file_hash:
+            return
+        db = SessionLocal()
+        try:
+            q = db.query(GalleryImage).filter(
+                GalleryImage.file_hash == file_hash,
+                GalleryImage.is_active == True,  # noqa: E712
+            )
+            if owner:
+                q = q.filter(GalleryImage.owner == owner)
+            img = q.first()
+            if not img:
+                return
+            img.caption = (text or "").strip()
+            db.commit()
+        except Exception as e:
+            db.rollback()
+            logger.warning("Failed to sync OCR caption to gallery image: %s", e)
+        finally:
+            db.close()
+
     @router.get("/{file_id}/vision")
     async def get_vision_text(request: Request, file_id: str, force: int = 0):
         """Return the vision-model OCR/description for an uploaded image.
@@ -289,7 +332,9 @@ def setup_upload_routes(upload_handler):
         if not force and os.path.exists(cache_path):
             try:
                 with open(cache_path, encoding="utf-8") as f:
-                    return {"text": f.read(), "cached": True}
+                    cached_text = f.read()
+                _sync_gallery_caption_for_upload(info, file_owner or current_user, cached_text)
+                return {"text": cached_text, "cached": True}
             except Exception as e:
                 logger.warning(f"Vision cache read failed for {file_id}: {e}")
         from src.document_processor import analyze_image_with_vl
@@ -303,6 +348,7 @@ def setup_upload_routes(upload_handler):
                 f.write(text)
         except Exception as e:
             logger.warning(f"Vision cache write failed for {file_id}: {e}")
+        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"text": text, "cached": False}
 
     @router.put("/{file_id}/vision")
@@ -333,6 +379,7 @@ def setup_upload_routes(upload_handler):
             raise HTTPException(400, "text must be a string")
         with open(_vision_cache_path(file_id), "w", encoding="utf-8") as f:
             f.write(text)
+        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"ok": True}
 
     async def periodic_rate_limit_cleanup():

scripts/odysseus-gallery

@@ -38,23 +38,27 @@ def _preview_text(value, limit: int = 200) -> str:
     return text[:limit]
 
 
+def _text_field(value) -> str:
+    return value if isinstance(value, str) else ""
+
+
 def _serialize_image(i: "GalleryImage") -> dict:
     return {
         "id": i.id,
-        "filename": i.filename,
+        "filename": _text_field(i.filename),
         "prompt": _preview_text(i.prompt),
-        "model": i.model or "",
-        "size": i.size or "",
-        "tags": i.tags or "",
+        "model": _text_field(i.model),
+        "size": _text_field(i.size),
+        "tags": _text_field(i.tags),
         "favorite": bool(i.favorite),
-        "album_id": i.album_id or "",
-        "session_id": i.session_id or "",
+        "album_id": _text_field(i.album_id),
+        "session_id": _text_field(i.session_id),
         "width": i.width,
         "height": i.height,
         "file_size": i.file_size,
         "taken_at": i.taken_at.isoformat() if i.taken_at else "",
-        "camera_make": i.camera_make or "",
-        "camera_model": i.camera_model or "",
+        "camera_make": _text_field(i.camera_make),
+        "camera_model": _text_field(i.camera_model),
         "created_at": i.created_at.isoformat() if i.created_at else "",
     }
 
@@ -93,11 +97,11 @@ def cmd_show(args):
         if not i:
             fail(f"no image with id {args.id!r}")
         out = _serialize_image(i)
-        out["prompt_full"] = i.prompt or ""
-        out["ai_tags"] = i.ai_tags or ""
+        out["prompt_full"] = _text_field(i.prompt)
+        out["ai_tags"] = _text_field(i.ai_tags)
         out["gps_lat"] = i.gps_lat or ""
         out["gps_lng"] = i.gps_lng or ""
-        out["file_hash"] = i.file_hash or ""
+        out["file_hash"] = _text_field(i.file_hash)
         emit(out, args)
     finally:
         db.close()

scripts/odysseus-mail

@@ -108,6 +108,8 @@ def _q(name: str) -> str:
 
 
 def _split_recipients(value: str) -> list[str]:
+    if not isinstance(value, str):
+        return []
     return [r.strip() for r in (value or "").split(",") if r.strip()]
 
 

scripts/odysseus-notes

@@ -36,7 +36,9 @@ def _load_items(raw) -> list:
         items = json.loads(raw)
     except (TypeError, json.JSONDecodeError):
         return []
-    return items if isinstance(items, list) else []
+    if not isinstance(items, list):
+        return []
+    return [item for item in items if isinstance(item, dict)]
 
 
 def _serialize(n: "Note") -> dict:

services/hwfit/data/hf_models.json

@@ -5113,8 +5113,8 @@
  {
   "name": "deepseek-ai/DeepSeek-V4-Flash",
   "provider": "deepseek-ai",
-  "parameter_count": "284B",
-  "parameters_raw": 284000000000,
+  "parameter_count": "158.1B",
+  "parameters_raw": 158069433298,
   "active_parameters": 13000000000,
   "is_moe": true,
   "min_ram_gb": 200.0,
@@ -5130,15 +5130,40 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 3542202,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 1882337,
+  "hf_likes": 1651,
+  "release_date": "2026-06-22"
+ },
+ {
+  "name": "deepseek-ai/DeepSeek-V4-Flash-DSpark",
+  "provider": "deepseek-ai",
+  "parameter_count": "165.3B",
+  "parameters_raw": 165265454782,
+  "active_parameters": 13000000000,
+  "is_moe": true,
+  "active_experts": 6,
+  "min_ram_gb": 170.0,
+  "recommended_ram_gb": 250.0,
+  "min_vram_gb": 165.0,
+  "quantization": "FP8-Mixed",
+  "context_length": 1000000,
+  "use_case": "General-purpose reasoning, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "deepseek_v4_moe",
+  "hf_downloads": 4446,
+  "hf_likes": 107,
+  "release_date": "2026-06-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Flash-Base",
   "provider": "deepseek-ai",
-  "parameter_count": "284B",
-  "parameters_raw": 284000000000,
+  "parameter_count": "292.0B",
+  "parameters_raw": 292021347282,
   "active_parameters": 13000000000,
   "is_moe": true,
   "min_ram_gb": 290.0,
@@ -5153,15 +5178,15 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 0,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 76030,
+  "hf_likes": 256,
+  "release_date": "2026-04-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Pro",
   "provider": "deepseek-ai",
-  "parameter_count": "1.6T",
-  "parameters_raw": 1600000000000,
+  "parameter_count": "861.6B",
+  "parameters_raw": 861608274846,
   "active_parameters": 49000000000,
   "is_moe": true,
   "min_ram_gb": 1100.0,
@@ -5177,15 +5202,40 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 0,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 1154610,
+  "hf_likes": 5118,
+  "release_date": "2026-06-22"
+ },
+ {
+  "name": "deepseek-ai/DeepSeek-V4-Pro-DSpark",
+  "provider": "deepseek-ai",
+  "parameter_count": "889.5B",
+  "parameters_raw": 889484881098,
+  "active_parameters": 49000000000,
+  "is_moe": true,
+  "active_experts": 6,
+  "min_ram_gb": 900.0,
+  "recommended_ram_gb": 1250.0,
+  "min_vram_gb": 890.0,
+  "quantization": "FP8-Mixed",
+  "context_length": 1000000,
+  "use_case": "Flagship reasoning, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "deepseek_v4_moe",
+  "hf_downloads": 6939,
+  "hf_likes": 241,
+  "release_date": "2026-06-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Pro-Base",
   "provider": "deepseek-ai",
   "parameter_count": "1.6T",
-  "parameters_raw": 1600000000000,
+  "parameters_raw": 1600790440862,
   "active_parameters": 49000000000,
   "is_moe": true,
   "min_ram_gb": 1700.0,
@@ -5200,9 +5250,9 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 0,
-  "hf_likes": 0,
-  "release_date": "2026-05-15"
+  "hf_downloads": 25387,
+  "hf_likes": 305,
+  "release_date": "2026-04-27"
  },
  {
   "name": "deepseek-ai/deepseek-coder-6.7b-base",
@@ -13308,6 +13358,106 @@
   "_discovered": true,
   "gguf_sources": []
  },
+ {
+  "name": "zai-org/GLM-5.2",
+  "provider": "zai-org",
+  "parameter_count": "753.3B",
+  "parameters_raw": 753329940480,
+  "min_ram_gb": 1510.0,
+  "recommended_ram_gb": 1800.0,
+  "min_vram_gb": 1510.0,
+  "quantization": "BF16",
+  "context_length": 1048576,
+  "use_case": "General purpose reasoning, coding, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "glm_moe_dsa",
+  "hf_downloads": 142547,
+  "hf_likes": 2996,
+  "release_date": "2026-06-23",
+  "is_moe": true,
+  "active_experts": 8,
+  "gguf_sources": [
+   {
+    "repo": "unsloth/GLM-5.2-GGUF",
+    "provider": "unsloth",
+    "file": "UD-Q4_K_M/*.gguf",
+    "quant": "Q4_K_M"
+   }
+  ]
+ },
+ {
+  "name": "zai-org/GLM-5.2-FP8",
+  "provider": "zai-org",
+  "parameter_count": "753.4B",
+  "parameters_raw": 753375793584,
+  "min_ram_gb": 760.0,
+  "recommended_ram_gb": 900.0,
+  "min_vram_gb": 760.0,
+  "quantization": "FP8",
+  "context_length": 1048576,
+  "use_case": "General purpose reasoning, coding, long-context",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "glm_moe_dsa",
+  "hf_downloads": 884226,
+  "hf_likes": 182,
+  "release_date": "2026-06-23",
+  "is_moe": true,
+  "active_experts": 8,
+  "gguf_sources": [
+   {
+    "repo": "unsloth/GLM-5.2-GGUF",
+    "provider": "unsloth",
+    "file": "UD-Q4_K_M/*.gguf",
+    "quant": "Q4_K_M"
+   }
+  ]
+ },
+ {
+  "name": "unsloth/GLM-5.2-GGUF",
+  "provider": "unsloth",
+  "parameter_count": "753.9B",
+  "parameters_raw": 753864139008,
+  "min_ram_gb": 452.0,
+  "recommended_ram_gb": 620.0,
+  "min_vram_gb": 452.0,
+  "quantization": "Q4_K_M",
+  "context_length": 1048576,
+  "use_case": "General purpose reasoning, coding, long-context (GGUF)",
+  "capabilities": [
+   "long_context",
+   "reasoning",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "text-generation",
+  "architecture": "glm-dsa",
+  "hf_downloads": 180394,
+  "hf_likes": 474,
+  "release_date": "2026-06-23",
+  "is_moe": true,
+  "active_experts": 8,
+  "is_gguf": true,
+  "gguf_sources": [
+   {
+    "repo": "unsloth/GLM-5.2-GGUF",
+    "provider": "unsloth",
+    "file": "UD-Q4_K_M/*.gguf",
+    "quant": "Q4_K_M"
+   }
+  ]
+ },
  {
   "name": "cyankiwi/Qwen3.5-35B-A3B-AWQ-4bit",
   "provider": "cyankiwi",
@@ -18955,6 +19105,54 @@
   "active_experts": 8,
   "active_parameters": 13600000000
  },
+ {
+  "name": "MiniMaxAI/MiniMax-M3",
+  "provider": "MiniMaxAI",
+  "parameter_count": "427.0B",
+  "parameters_raw": 427040140160,
+  "min_ram_gb": 855.0,
+  "recommended_ram_gb": 1025.0,
+  "min_vram_gb": 855.0,
+  "quantization": "BF16",
+  "context_length": 1000000,
+  "use_case": "Vision, chat, coding, agentic tool use",
+  "capabilities": [
+   "vision",
+   "tool_use",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "image-text-to-text",
+  "architecture": "minimax_m3_vl",
+  "hf_downloads": 192311,
+  "hf_likes": 1267,
+  "release_date": "2026-06-23",
+  "is_moe": true
+ },
+ {
+  "name": "MiniMaxAI/MiniMax-M3-MXFP8",
+  "provider": "MiniMaxAI",
+  "parameter_count": "440.3B",
+  "parameters_raw": 440279845760,
+  "min_ram_gb": 445.0,
+  "recommended_ram_gb": 560.0,
+  "min_vram_gb": 445.0,
+  "quantization": "MXFP8",
+  "context_length": 1000000,
+  "use_case": "Vision, chat, coding, agentic tool use",
+  "capabilities": [
+   "vision",
+   "tool_use",
+   "coding",
+   "moe"
+  ],
+  "pipeline_tag": "image-text-to-text",
+  "architecture": "minimax_m3_vl",
+  "hf_downloads": 572278,
+  "hf_likes": 43,
+  "release_date": "2026-06-15",
+  "is_moe": true
+ },
  {
   "name": "bullerwins/MiniMax-M2.7-REAP-172B-fp8",
   "provider": "bullerwins",
@@ -19276,4 +19474,4 @@
   ],
   "_discovered": true
  }
-]
+]

services/hwfit/profiles.py

@@ -103,6 +103,9 @@ def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=Non
     in the actual serving knobs (n_cpu_moe, KV-cache type, context). serve_quant
     is the file's quant label (e.g. "Q4_K_M") just for display.
     """
+    if not isinstance(system, dict) or not isinstance(model, dict):
+        return []
+
     vram = float(system.get("gpu_vram_gb") or 0)
     if vram <= 0:
         return []

services/search/content.py

@@ -8,11 +8,13 @@ import os
 import re
 import logging
 import socket
+import ssl
 from datetime import datetime, timedelta
-from typing import List
+from typing import Iterable, List, cast
 from urllib.parse import urljoin, urlparse
 
 import httpx
+import httpcore
 from bs4 import BeautifulSoup
 
 from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
@@ -91,6 +93,148 @@ def _public_http_url(url: str) -> bool:
         return False
 
 
+def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname:
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    host = (parsed.hostname or "").strip().lower()
+    if host in ("localhost", "metadata", "metadata.google.internal"):
+        raise httpx.RequestError(f"Blocked non-public hostname: {host}")
+    try:
+        ip = ipaddress.ip_address(host)
+        if _is_private_address(ip):
+            raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
+        return [ip]
+    except httpx.RequestError:
+        raise
+    except ValueError:
+        pass
+    addrs = _resolve_hostname_ips(host)
+    if not addrs or any(_is_private_address(a) for a in addrs):
+        raise httpx.RequestError(f"Blocked non-public URL: {url}")
+    return addrs
+
+
+class _PinnedBackend(httpcore.NetworkBackend):
+    """Network backend that connects to a pre-resolved IP.
+
+    httpcore derives the TLS SNI and the ``Host`` header from the URL's
+    origin, not from the host argument passed to ``connect_tcp``. So
+    routing the TCP connect to a resolved IP while leaving the URL
+    untouched keeps SNI / vhost behaviour correct and closes the
+    DNS-rebinding TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress):
+        self._ip = str(ip)
+        self._real = httpcore.SyncBackend()
+
+    def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: float | None = None,
+        local_address: str | None = None,
+        socket_options=None,
+    ):
+        return self._real.connect_tcp(
+            self._ip, port, timeout, local_address, socket_options
+        )
+
+    def connect_unix_socket(self, path, timeout=None, socket_options=None):
+        return self._real.connect_unix_socket(path, timeout, socket_options)
+
+    def sleep(self, seconds: float) -> None:
+        return self._real.sleep(seconds)
+
+
+# Map httpcore exception classes to their httpx equivalents. Built
+# once at import time from the public exception classes; avoids any
+# import of httpx's private transport machinery. httpcore's
+# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
+# close and retry on its own) — we never expect to see it surface to
+# a transport caller, so it has no httpx counterpart here.
+_HTTPCORE_TO_HTTPX_EXC = {
+    httpcore.ConnectError: httpx.ConnectError,
+    httpcore.ConnectTimeout: httpx.ConnectTimeout,
+    httpcore.LocalProtocolError: httpx.LocalProtocolError,
+    httpcore.NetworkError: httpx.NetworkError,
+    httpcore.PoolTimeout: httpx.PoolTimeout,
+    httpcore.ProtocolError: httpx.ProtocolError,
+    httpcore.ProxyError: httpx.ProxyError,
+    httpcore.ReadError: httpx.ReadError,
+    httpcore.ReadTimeout: httpx.ReadTimeout,
+    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
+    httpcore.TimeoutException: httpx.TimeoutException,
+    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
+    httpcore.WriteError: httpx.WriteError,
+    httpcore.WriteTimeout: httpx.WriteTimeout,
+}
+
+
+class _PinnedTransport(httpx.BaseTransport):
+    """Transport that pins every TCP connect to a pre-resolved IP.
+
+    Uses only the public ``httpcore`` and ``httpx`` APIs — no
+    subclassing of ``httpx.HTTPTransport``, no reads of private
+    ``httpcore.ConnectionPool`` attributes, no imports from
+    ``httpx private transport internals``. The URL is passed through unchanged so SNI
+    / vhost work as if httpx had been given the hostname directly;
+    only the TCP destination is pinned, closing the DNS-rebinding
+    TOCTOU between the SSRF check and the connect.
+    """
+
+    def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
+        self._pool = httpcore.ConnectionPool(
+            ssl_context=ssl.create_default_context(),
+            http1=True,
+            http2=http2,
+            network_backend=_PinnedBackend(ip),
+        )
+
+    def __enter__(self):
+        self._pool.__enter__()
+        return self
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
+        self._pool.__exit__(exc_type, exc_value, traceback)
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        httpcore_req = httpcore.Request(
+            method=request.method,
+            url=httpcore.URL(
+                scheme=request.url.raw_scheme,
+                host=request.url.raw_host,
+                port=request.url.port,
+                target=request.url.raw_path,
+            ),
+            headers=request.headers.raw,
+            content=request.stream,
+            extensions=request.extensions,
+        )
+        try:
+            httpcore_resp = self._pool.handle_request(httpcore_req)
+            # Eager materialisation matches the original
+            # ``response.text`` usage in fetch_webpage_content. The
+            # sync pool's stream is a plain Iterable[bytes] despite
+            # the httpcore type hint unioning the async variant.
+            content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
+        except Exception as exc:
+            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
+            if mapped is not None:
+                raise mapped(str(exc)) from exc
+            raise
+
+        return httpx.Response(
+            status_code=httpcore_resp.status,
+            headers=httpcore_resp.headers,
+            content=content,
+            extensions=httpcore_resp.extensions,
+        )
+
+    def close(self) -> None:
+        self._pool.close()
+
 class BodyTooLargeError(Exception):
     """The server declared a body larger than the hard fetch ceiling."""
 
@@ -141,78 +285,78 @@ class _CappedFetch:
 
 def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
                     max_bytes: int = None) -> "_CappedFetch":
-    """Capped streaming GET with SSRF-guarded manual redirects.
+    """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
 
-    The body is streamed and buffering stops at ``max_bytes`` (default: the
-    soft cap), so an oversized resource cannot be pulled into memory or the
-    content cache in full. When Content-Length already declares a body over
-    the hard ceiling, the fetch is refused before any body bytes are read.
+    Each hop is resolved once, validated as public, and then the actual TCP
+    connection is pinned to that resolved IP. The request URL is left unchanged
+    so Host and TLS SNI keep the original hostname.
     """
     cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
     current = url
     for _ in range(max_redirects + 1):
-        if not _public_http_url(current):
-            raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
+        ips = _resolve_public_ips(current)
+
         # Force identity transfer-encoding. With gzip/deflate the wire bytes
-        # (and Content-Length) can be a small fraction of the decoded body, so
-        # a tiny compressed response could pass the hard-cap preflight and then
-        # expand past the ceiling in a single decoded chunk before the streamed
-        # cap below can slice it. Identity makes Content-Length the true body
-        # size and keeps each streamed chunk bounded by the network read.
+        # and Content-Length can be a small fraction of the decoded body, so a
+        # tiny compressed response could pass the hard-cap preflight and then
+        # expand past the ceiling in one decoded chunk before the streamed cap
+        # below can slice it.
         req_headers = dict(headers or {})
         req_headers["Accept-Encoding"] = "identity"
-        with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
-                          follow_redirects=False) as response:
-            if response.status_code in (301, 302, 303, 307, 308):
-                location = response.headers.get("location")
-                if not location:
-                    return _CappedFetch(response.status_code, response.headers, b"",
-                                        False, None, response.encoding, str(response.url))
-                current = urljoin(str(response.url), location)
-                continue
 
-            # A server can ignore the identity request and still return a
-            # compressed body; httpx.iter_bytes would then decode it, and a tiny
-            # gzip can balloon into one decoded chunk far past the cap before we
-            # slice. Refuse a compressed Content-Encoding so the streamed cap
-            # stays a real memory bound (Content-Length is the compressed wire
-            # length here, so the preflight and size metadata are unreliable too).
-            enc = (response.headers.get("content-encoding") or "").strip().lower()
-            if enc and enc != "identity":
-                raise httpx.RequestError(
-                    f"Refusing compressed response (Content-Encoding: {enc}) after "
-                    "requesting identity: cannot bound decoded body size",
-                    request=httpx.Request("GET", current),
-                )
+        with httpx.Client(
+            headers=req_headers,
+            timeout=timeout,
+            follow_redirects=False,
+            transport=_PinnedTransport(ips[0]),
+        ) as client:
+            with client.stream("GET", current) as response:
+                if response.status_code in (301, 302, 303, 307, 308):
+                    location = response.headers.get("location")
+                    if not location:
+                        return _CappedFetch(response.status_code, response.headers, b"",
+                                            False, None, response.encoding, str(response.url))
+                    current = urljoin(str(response.url), location)
+                    continue
 
-            declared = None
-            raw_len = response.headers.get("content-length")
-            if raw_len and raw_len.isdigit():
-                declared = int(raw_len)
-            # Refuse before buffering anything when the server already tells
-            # us the body exceeds the absolute ceiling (Content-Length is wire
-            # bytes; the decompressed body can only be larger).
-            if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
-                raise BodyTooLargeError(current, declared)
+                # A server can ignore the identity request and still return a
+                # compressed body; httpx.iter_bytes would then decode it, and a
+                # tiny gzip can balloon into one decoded chunk far past the cap.
+                # Refuse compressed Content-Encoding so the streamed cap stays
+                # a real memory bound.
+                enc = (response.headers.get("content-encoding") or "").strip().lower()
+                if enc and enc != "identity":
+                    raise httpx.RequestError(
+                        f"Refusing compressed response (Content-Encoding: {enc}) after "
+                        "requesting identity: cannot bound decoded body size",
+                        request=httpx.Request("GET", current),
+                    )
+
+                declared = None
+                raw_len = response.headers.get("content-length")
+                if raw_len and raw_len.isdigit():
+                    declared = int(raw_len)
+
+                if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
+                    raise BodyTooLargeError(current, declared)
+
+                chunks = []
+                read = 0
+                truncated = False
+                for chunk in response.iter_bytes():
+                    read += len(chunk)
+                    if read > cap:
+                        keep = cap - (read - len(chunk))
+                        if keep > 0:
+                            chunks.append(chunk[:keep])
+                        truncated = True
+                        break
+                    chunks.append(chunk)
+
+                return _CappedFetch(response.status_code, response.headers,
+                                    b"".join(chunks), truncated, declared,
+                                    response.encoding, str(response.url))
 
-            chunks = []
-            read = 0
-            truncated = False
-            # We requested identity above, so iter_bytes yields the raw body in
-            # network-read-sized chunks (no decompression expansion); the cap
-            # therefore bounds what we actually buffer.
-            for chunk in response.iter_bytes():
-                read += len(chunk)
-                if read > cap:
-                    keep = cap - (read - len(chunk))
-                    if keep > 0:
-                        chunks.append(chunk[:keep])
-                    truncated = True
-                    break
-                chunks.append(chunk)
-            return _CappedFetch(response.status_code, response.headers,
-                                b"".join(chunks), truncated, declared,
-                                response.encoding, str(response.url))
     raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
 
 # PDF extraction (optional dependency)

src/agent_loop.py

@@ -15,7 +15,11 @@ import logging
 from typing import AsyncGenerator, List, Dict, Optional, Set
 from urllib.parse import urlparse
 
-from src.llm_core import stream_llm, stream_llm_with_fallback, _is_ollama_native_url
+from src.llm_core import (
+    stream_llm,
+    stream_llm_with_fallback,
+    _is_ollama_native_url,
+)
 from src.model_context import estimate_tokens
 from src.settings import get_setting
 from src.prompt_security import untrusted_context_message
@@ -130,7 +134,8 @@ _API_AGENT_RULES = """\
 - "Disable/turn off/enable/turn on <tool>" (shell, search, research, browser, documents, incognito, etc.) → call `ui_control` with `toggle <name> <on|off>`. Aliases accepted: shell→bash, search→web, deepresearch→research, documents→document_editor. NEVER record this as a memory — the user wants the toggle flipped, not a note about preferring it.
 - "Research X" / "do research on X" / "look into Y" / "deep dive on Z" → call `trigger_research` with `topic`. This starts a live job that appears in the Deep Research sidebar (streams progress + final report). **Do NOT use `web_search` for these** — saw the agent do a plain web_search for "do research on X" when the user wanted the deep-research job. "research X" is a deep-research request, not a quick lookup. (web_search is only for a single quick fact mid-task.) Do NOT POST /api/research/start via app_api either — blocked. After starting, tell the user it's running in the Deep Research sidebar. Only if the user explicitly wants it inline/quick should you fall back to web_search.
 - "Open/show <panel>" (documents, library, gallery, email, inbox, sessions, brain/memories, skills, settings, notes, cookbook) → call `ui_control` with `open_panel <name>`. Panel aliases: library/doc/docs/document→documents, images→gallery, mail/inbox/emails→email, chats/history→sessions, memory/memories→brain, preferences→settings, models/serve/serving→cookbook. CRITICAL: "open memory/memories/brain" / "open skills" / "open notes" / "open documents" / "open cookbook" means OPEN THE PANEL — call `ui_control`, NOT a manage/list tool. The "manage_*" tools list contents in chat; `ui_control open_panel` opens the visual modal the user is asking for.
-- "Open/start a reply", "open a reply to <sender>", "draft a reply window" for email → find/read the email if needed, then call `ui_control` with `open_email_reply <uid> <folder> reply`. This opens the same email document compose window as clicking Reply in the Email UI. Do NOT call `reply_to_email` unless the user explicitly gave body text and wants to SEND immediately.
+- "Write/draft a reply saying X" for an open/read email → call `ui_control` with `action="open_email_reply"`, the email `uid`/`folder`, `mode="reply"`, and `body` containing the drafted reply. This opens the same email compose document as clicking Reply and DOES NOT send. Do NOT call `reply_to_email` unless the user explicitly says to send immediately.
+- "Open/start a reply", "open a reply to <sender>", "draft a reply window" with no requested body → find/read the email if needed, then call `ui_control` with `open_email_reply <uid> <folder> reply`.
 - Bulk email actions ("delete all those", "archive these", "mark all read") require a real email tool call. Use `bulk_email` once with UIDs from the latest `list_emails` result and the same `account`; never claim success without the tool result.
 - Email UIDs are the values after `UID:` in tool output, not list row numbers. For example, row `1.` with `UID: 90186` must use `"90186"`, never `"1"`.
 - "Last/latest/newest email" means call `list_emails` with `max_results: 1`, `unread_only: false`, and the right `account`, then read the UID returned by that tool if full content is needed. NEVER use a table row number like "#18" as an email UID.
@@ -230,7 +235,7 @@ _DOMAIN_RULES = {
 - For latest/newest email, list with `max_results: 1`, `unread_only: false`, then read the returned UID if needed.
 - For named mailboxes/accounts, call `list_email_accounts` if needed and pass the exact `account` value.
 - Bulk email actions use `bulk_email` once with explicit UIDs; do not loop one message at a time.
-- "Open/start a reply" means open a draft via `ui_control open_email_reply`; only `reply_to_email` when the user clearly wants to send now.""",
+- "Write/draft a reply saying X" means open a pre-filled draft via `ui_control open_email_reply ... <body>` / structured `body`; only `reply_to_email` when the user clearly wants to send now.""",
     "cookbook": """\
 ## Cookbook/model-serving rules
 - Cookbook is the LLM-serving subsystem.
@@ -446,7 +451,7 @@ List recent emails from a folder, newest first, including read messages by defau
 ```reply_to_email
 {"uid": "1234", "body": "Sounds good — talk Friday.", "account": "gmail"}
 ```
-SEND a reply email immediately by UID. Do not use this for "open a reply" or "start a reply" — those should use `ui_control` with `open_email_reply <uid> <folder> reply` to open the email draft document. For follow-up requests like "reply ..." after reading/listing email where the user clearly wants to send now, use the exact UID and account from the latest `read_email`/`list_emails` result. Never invent UID `1`. Threads automatically (In-Reply-To/References handled).
+SEND a reply email immediately by UID. Do not use this for "write/draft a reply", "open a reply", or "start a reply" — those should use `ui_control` with `open_email_reply <uid> <folder> reply <body>` (or structured `body`) to open the email draft document. Only use this when the user explicitly says to send now. Never invent UID `1`. Threads automatically (In-Reply-To/References handled).
 
 CRITICAL — signatures: DO NOT invent a sign-off name. End the body with just `Thanks,` or similar — never type a person's name unless the user explicitly told you what to sign as. When `agent_email_confirm` is on (default), the tool returns `{pending: true, pending_id: ...}` and stages the email for the user to approve in the chat UI instead of SMTPing immediately.""",
     "bulk_email": """\
@@ -466,9 +471,10 @@ Bulk delete/archive/mark emails. Use this for "delete all those" after listing e
 Calendar event management (CalDAV). Actions: `list_events`, `create_event`, `update_event`, `delete_event`, `list_calendars`. \
 For `list_events`: {start?, end?, calendar?}; prefer `start`/`end` for the range, though start_date/end_date and from/to aliases are accepted. \
 For `create_event`: {summary, dtstart, dtend?, duration?, calendar?, location?, description?, reminder_minutes?, rrule?}. \
+For `update_event`: {uid, summary?, dtstart?, dtend?, all_day?, location?, description?, event_type?, importance?, rrule?}. Pass `rrule: ""` to remove recurrence and make a repeating event a single event. \
 `dtstart` accepts natural language ("tomorrow at 1pm", "in 2 hours", "next monday 9am") or ISO ("2026-05-12T13:00:00"). \
 If `dtend` omitted, defaults to dtstart+1h (or +1d when `all_day: true`). \
-For a RECURRING event pass `rrule` as an iCalendar RRULE string, e.g. `"FREQ=WEEKLY;BYDAY=MO"` (every Monday), `"FREQ=DAILY;COUNT=10"`, or `"FREQ=MONTHLY;BYMONTHDAY=1"` — create ONE event with the rrule, do not loop creating many events. \
+For a RECURRING event pass `rrule` as an iCalendar RRULE string, e.g. `"FREQ=WEEKLY;BYDAY=MO"` (every Monday), `"FREQ=DAILY;COUNT=10"`, or `"FREQ=MONTHLY;BYMONTHDAY=1"` — create ONE event with the rrule, do not loop creating many events. Do not pass `rrule` for "next Wednesday only", "just this once", or any single occurrence. \
 If the user asks for a reminder/alarm before the event, pass `reminder_minutes` as an integer; do not write reminder text into the event description and do NOT also call `manage_notes` for the same reminder because calendar reminders are routed through Notes automatically. \
 `calendar` accepts a name ("Main") or short-id prefix.""",
     "create_session": "- ```create_session``` — Create a new chat. Line 1 = chat name, line 2 = model name. Use for background/parallel work.",
@@ -476,7 +482,7 @@ If the user asks for a reminder/alarm before the event, pass `reminder_minutes`
     "send_to_session": "- ```send_to_session``` — Send a message to another session. Line 1 = session_id, rest = message. Use for orchestrating work across sessions.",
     "search_chats": "- ```search_chats``` — Search past session transcripts for direct conversation evidence. Use when user asks 'did we discuss X?', 'find the conversation about Y', or when prior chat context is more appropriate than persistent memory.",
     "pipeline": "- ```pipeline``` — Run a multi-step AI pipeline. Args (JSON) with ordered steps, each specifying a model and prompt. Use for complex workflows.",
-    "ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle <name> on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel <name>` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply <uid> <folder> <reply|reply-all|ai-reply>` (opens an email compose document, does NOT send), `set_mode agent/chat`, `switch_model <name>`, `set_theme <preset>`, `create_theme <name> <bg> <fg> <panel> <border> <accent>` (optional key=val for advanced colors AND background effects: bgPattern=<none|dots|synapse|rain|constellations|perlin-flow|petals|sparkles|embers>, bgEffectColor=#RRGGBB, bgEffectIntensity=<num>, bgEffectSize=<num>, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel <name>`. Built-in theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute. For any other vibe/name, use create_theme.",
+    "ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle <name> on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel <name>` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply <uid> <folder> <reply|reply-all|ai-reply> <body text>` (opens an email compose document pre-filled with body, DOES NOT send; use this for normal “write/draft a reply saying X” requests), `set_mode agent/chat`, `switch_model <name>`, `set_theme <preset>`, `create_theme <name> <bg> <fg> <panel> <border> <accent>` (optional key=val for advanced colors AND background effects: bgPattern=<none|dots|synapse|rain|constellations|perlin-flow|petals|sparkles|embers>, bgEffectColor=#RRGGBB, bgEffectIntensity=<num>, bgEffectSize=<num>, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel <name>`. Built-in theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute. For any other vibe/name, use create_theme.",
     "ask_user": "- ```ask_user``` — Ask the user a multiple-choice question when the task is genuinely ambiguous and the answer changes what you do next (pick an approach, confirm an assumption, choose a target). Args (JSON): {\"question\": \"...\", \"options\": [{\"label\": \"...\", \"description\": \"...\"?}, ...], \"multi\": false?}. 2-6 options. The user gets clickable buttons; calling this ENDS your turn and their choice comes back as your next message. Prefer sensible defaults — only ask when you truly can't proceed well without their input.",
     "update_plan": "- ```update_plan``` — While executing an approved plan, write the plan back: tick steps done or revise them. Args (JSON): {\"plan\": \"- [x] done step\\n- [ ] next step\"}. Always pass the COMPLETE checklist, not a diff. Call it after finishing each step (mark it `- [x]`) and whenever the user asks to change the plan. The user's docked plan window updates live. Does nothing if there's no active plan.",
     "list_served_models": "- ```list_served_models``` — Show what the Cookbook (LLM-serving subsystem) is currently running. NO args. Use this for ANY 'what's running' / 'what's serving' / 'show my cookbook' / 'is anything up' query. DO NOT shell out (`ps aux`, `docker ps`, etc.) — this tool is the source of truth. Failed serve tasks include recent logs plus diagnosis/retry suggestions; use those suggestions to call `serve_model` again with an adjusted command when appropriate.",
@@ -574,11 +580,13 @@ def _assemble_prompt(tool_names: set, disabled_tools: set = None, compact: bool
         tool_lines = []
         for name, _default_section in TOOL_SECTIONS.items():
             if name in included:
-                tool_lines.append(_compact_tool_line(name, _section_text(name, _default_section)))
+                tool_lines.append(f"- `{name}`")
         parts = [
-            _AGENT_PREAMBLE,
+            "You are an AI assistant with native tool/function calling. "
+            "Only the tool schemas provided by the API are available for this turn. "
+            "Use native tool calls when action is needed; do not write tool syntax or tool instructions in chat.",
             "## Available tools\n" + ("\n".join(tool_lines) if tool_lines else "none"),
-            _AGENT_RULES,
+            _API_AGENT_RULES,
         ]
         parts.extend(_domain_rules_for_tools(included))
         return "\n\n".join(parts)
@@ -971,6 +979,11 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o
         domains.add("notes_calendar_tasks")
     if has(r"\b(calendar|event|meeting|appointment|schedule)\b"):
         domains.add("notes_calendar_tasks")
+    _code_write_intent = has(
+        r"\b(?:python|javascript|typescript|java|c\+\+|cpp|c#|csharp|rust|go|golang|"
+        r"ruby|php|swift|kotlin|bash|shell|html|css|sql)\b",
+        r"\b(?:code|script|program|game|function|class|module|app)\b",
+    )
     if has(r"\b(documents?|docs?|draft|compose|poem|story|essay|outline|letter|edit|rewrite|proofread|suggest|feedback|review this|make a file)\b"):
         domains.add("documents")
     if "notes_calendar_tasks" not in domains and has(r"\bwrite\b"):
@@ -989,7 +1002,18 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o
         domains.add("ui")
     if has(r"\b(session|chat history|rename chat|delete chat|archive chat|fork chat|list chats)\b"):
         domains.add("sessions")
-    if has(r"\b(file|folder|directory|repo|git|grep|find in files|read file|edit file|shell|terminal|bash|python)\b"):
+    if has(r"\b(file|folder|directory|repo|git|grep|find in files|read file|edit file|shell|terminal|bash)\b"):
+        domains.add("files")
+    if has(
+        r"\b(run|execute|test|debug|fix|save|create|edit|read|open)\b.{0,40}\b("
+        r"python|javascript|typescript|java|c\+\+|cpp|c#|csharp|rust|go|golang|"
+        r"ruby|php|swift|kotlin|bash|shell|html|css|sql|code|script|program|game"
+        r")\b",
+        r"\b("
+        r"python|javascript|typescript|java|c\+\+|cpp|c#|csharp|rust|go|golang|"
+        r"ruby|php|swift|kotlin|bash|shell|html|css|sql"
+        r")\b.{0,40}\b(file|script|program|app)\b",
+    ):
         domains.add("files")
     # Managing detached bash jobs: "kill the background job", "stop the job",
     # "kill that job", "check the job output", "is the bg job done".
@@ -1020,6 +1044,224 @@ def _classify_agent_request(messages: List[Dict], last_user: str) -> Dict[str, o
     }
 
 
+def _turn_targets_active_document(intent: Dict[str, object], last_user: str, active_document) -> bool:
+    """Return whether an open document should affect this turn.
+
+    The editor can stay open while the user asks unrelated things ("who am I?",
+    "search news"). In those cases injecting document context/tools makes small
+    models overfit to the visible document and call suggest/edit tools. Keep the
+    active document only for explicit document domains or common document-edit
+    continuations.
+    """
+    if active_document is None:
+        return False
+    raw_doc = getattr(active_document, "current_content", "") or ""
+    title_l = (getattr(active_document, "title", "") or "").strip().lower()
+    is_email_doc = (
+        getattr(active_document, "language", None) == "email"
+        or title_l in {"new email", "new mail", "new message"}
+        or ("To:" in raw_doc[:400] and "Subject:" in raw_doc[:400] and "\n---\n" in raw_doc)
+    )
+    if "documents" in (intent.get("domains") or set()):
+        return True
+    text = str(last_user or "").strip().lower()
+    if not text:
+        return False
+    if is_email_doc and re.search(
+        r"\b("
+        r"email|mail|reply|respond|response|draft|compose|send|"
+        r"tell them|tell her|tell him|say|write|make it say|"
+        r"japanese|japan|polite|formal|tone|style"
+        r")\b",
+        text,
+    ):
+        return True
+    if re.search(
+        r"\b(?:add|insert|include|apply|put)\b.+\b(?:to it|to this|there|in it|in this|in the text|in the document)\b",
+        text,
+    ):
+        return True
+    if re.search(
+        r"\b(?:make it|make this|expand it|expand this|extend it|extend this|continue it|continue this)\b.*\b(?:longer|shorter|bigger|smaller|more detailed|more concise|expanded|extended)?\b",
+        text,
+    ):
+        return True
+    return bool(re.search(
+        r"\b("
+        r"document|doc|draft|text|poem|story|essay|outline|letter|paragraph|"
+        r"stanza|line|title|heading|section|sentence|word|caps|uppercase|"
+        r"lowercase|rewrite|reword|style|tone|suggest|suggestions|feedback|"
+        r"improve|edit|change|remove|delete|replace|add another|append|"
+        r"original text|in the document|the document|this document"
+        r")\b",
+        text,
+    ))
+
+
+def _minimal_saved_memory_message(messages: List[Dict]) -> Optional[Dict]:
+    facts: List[str] = []
+    seen = set()
+    for message in messages:
+        if not isinstance(message, dict):
+            continue
+        metadata = message.get("metadata") if isinstance(message, dict) else None
+        source = str((metadata or {}).get("source") or "")
+        if not source.startswith("saved memory:"):
+            continue
+        content = str(message.get("content") or "")
+        content = re.sub(r"(?m)^\s*Source:\s*saved memory:[^\n]*\n?", "", content)
+        content = content.replace("Core facts about the user:", "")
+        content = re.sub(
+            r"Memory context\. Do not reference unless the user asks about these topics\.\s*",
+            "",
+            content,
+        )
+        for line in content.splitlines():
+            line = line.strip()
+            if not line.startswith("- "):
+                continue
+            fact = line[2:].strip()
+            if not fact or fact in seen:
+                continue
+            seen.add(fact)
+            facts.append(fact)
+            if len(facts) >= 12:
+                break
+        if len(facts) >= 12:
+            break
+    if not facts:
+        return None
+    logger.info("[agent-intent] odysseus doc minimal memory facts=%s", len(facts))
+    return {
+        "role": "user",
+        "content": (
+            "Saved user memory facts from Odysseus Brain. These are the same "
+            "user facts available in the normal prompt path. Use them when "
+            "the user asks for personalization, identity, background, "
+            "preferences, or anything about \"me\" or \"my\":\n"
+            + "\n".join(f"- {fact}" for fact in facts)
+        ),
+    }
+
+
+def _minimal_odysseus_doc_messages(messages: List[Dict], active_document, stream_create: bool = False) -> List[Dict]:
+    """Tiny prompt path for the Odysseus document LoRA.
+
+    This model is trained on document tool behavior, so avoid the normal agent
+    rule stack and send only the task plus the active document when editing.
+    """
+    latest = _extract_last_user_message(messages)
+    if stream_create:
+        system = (
+            "You are Odysseus. Create the requested document by streaming exactly one fenced block:\n"
+            "```document\n"
+            "Title\n"
+            "markdown\n"
+            "Document content\n"
+            "```\n"
+            "Do not use native function-call JSON or <tool_calls> markup. "
+            "Use only the fenced document block above. Do not write anything before the fence. "
+            "Use saved user memory facts when the user asks for something relating to them."
+        )
+    else:
+        system = (
+            "You are Odysseus. Edit or suggest changes to the active document using exactly one fenced tool block when needed.\n"
+            "If the user asks to add, remove, rewrite, transform, change, capitalize, shorten, expand, or otherwise apply a change, use edit_document or update_document, not suggest_document.\n"
+            "Use suggest_document only when the user explicitly asks for suggestions, feedback, or proposed improvements without applying them.\n"
+            "For targeted edits:\n"
+            "```edit_document\n"
+            "<<<FIND>>>\n"
+            "exact text from the active document\n"
+            "<<<REPLACE>>>\n"
+            "replacement text\n"
+            "<<<END>>>\n"
+            "```\n"
+            "For full rewrites only:\n"
+            "```update_document\n"
+            "entire new document content\n"
+            "```\n"
+            "For improvement suggestions:\n"
+            "```suggest_document\n"
+            "<<<FIND>>>\n"
+            "text to improve\n"
+            "<<<SUGGEST>>>\n"
+            "suggested replacement\n"
+            "<<<REASON>>>\n"
+            "why this improves it\n"
+            "<<<END>>>\n"
+            "```\n"
+            "Do not use native function-call JSON or <tool_calls> markup. "
+            "FIND text must be copied exactly from the active document with no labels like content:, title:, or markdown. "
+            "Use only the fenced tool blocks above. Do not write anything before the fenced block. "
+            "After the tool succeeds, Odysseus will answer Done."
+        )
+    out = [{"role": "system", "content": system}]
+    memory_message = _minimal_saved_memory_message(messages)
+    if memory_message:
+        out.append(memory_message)
+    if active_document is not None:
+        content = active_document.current_content or ""
+        out.append({
+            "role": "user",
+            "content": (
+                "Active document:\n"
+                f"Title: {active_document.title}\n"
+                f"Language: {active_document.language or 'text'}\n"
+                "Content:\n"
+                f"{content}"
+            ),
+        })
+    out.append({"role": "user", "content": latest})
+    return out
+
+
+_DOC_MODEL_ARTIFACT_RE = re.compile(
+    r"(?:\|end\|)+\|?assistan(?:t)?\|?"
+    r"|\|assistan(?:t)?\|"
+    r"|<\|im_start\|>\s*assistant"
+    r"|<\|im_end\|>",
+    re.IGNORECASE,
+)
+
+
+def _strip_doc_model_artifacts(text: str) -> str:
+    return _DOC_MODEL_ARTIFACT_RE.sub("", text or "")
+
+
+def _normalize_stream_document_fences(text: str, target_tool: str = "create_document") -> str:
+    """Treat visible ```document/documen blocks as document tool blocks.
+
+    The document LoRA occasionally emits a neutral/truncated `documen` fence.
+    For new documents that maps to create_document. For active-document turns,
+    the same shape is a full replacement of the open document, so map it to
+    update_document and drop the title/language header lines.
+    """
+    text = _strip_doc_model_artifacts(text or "")
+
+    def repl(match: re.Match) -> str:
+        body = match.group(1) or ""
+        if target_tool == "update_document":
+            lines = body.splitlines()
+            if lines and not lines[0].lstrip().startswith("#"):
+                lines = lines[1:]
+            if lines and lines[0].strip().lower() in {
+                "markdown", "md", "text", "txt", "html", "email",
+                "python", "javascript", "typescript", "json", "yaml",
+            }:
+                lines = lines[1:]
+            while lines and not lines[0].strip():
+                lines = lines[1:]
+            body = "\n".join(lines)
+        return f"```{target_tool}\n{body}"
+
+    return re.sub(
+        r"```documen(?:t)?\s*\n([\s\S]*?)(?=\n```|$)",
+        repl,
+        text,
+        flags=re.IGNORECASE,
+    )
+
+
 def _recent_context_for_retrieval(messages: List[Dict], max_user: int = 3, max_chars: int = 600) -> str:
     """Build the tool-retrieval query from the last few USER turns, not just
     the latest one.
@@ -1142,9 +1384,18 @@ def _build_system_prompt(
     # the trusted system role. Bound up front so the insert block below can
     # always check it.
     _skills_message = None
+    _email_style_message = None
+    _integ_message = None
+    _mcp_desc_message = None
     if active_document:
         set_active_document(active_document.id)
         _doc_raw = active_document.current_content or ""
+        _document_writing_style = ""
+        try:
+            from src.settings import load_settings as _load_settings
+            _document_writing_style = (_load_settings().get("document_writing_style", "") or "").strip()
+        except Exception:
+            _document_writing_style = ""
         _doc_title_l = (active_document.title or "").strip().lower()
         _is_email_doc = (
             active_document.language == "email"
@@ -1235,6 +1486,21 @@ def _build_system_prompt(
                     f'text must match the document EXACTLY and must NOT include the leading line-number '
                     f'or tab (those are reference-only). To rewrite entirely: update_document.'
                 )
+                if _document_writing_style:
+                    doc_ctx += (
+                        "\n\nDOCUMENT WRITING STYLE — use only for normal prose writing/revision in this "
+                        "document, not for code/data/JSON and not for email-specific greetings or signatures:\n"
+                        f"{_document_writing_style}"
+                    )
+                else:
+                    doc_ctx += (
+                        "\n\nStyle safety: if the user asks to write/rewrite this document \"in my style\" "
+                        "or \"as my style\", do NOT infer that style from memories, identity, public persona, "
+                        "creator/channel references, or biographical facts. There is no saved document writing "
+                        "style. Ask the user for a style sample or a document writing style description before "
+                        "rewriting for style. You may still make ordinary requested edits that do not depend on "
+                        "knowing the user's personal style."
+                    )
         _doc_message = untrusted_context_message("active editor document", doc_ctx)
         _doc_message["_protected"] = True
 
@@ -1291,10 +1557,11 @@ def _build_system_prompt(
             f"answer is ALWAYS the sender of the open email (above) unless they "
             f"named someone else. Asking that is the wrong move every time.\n\n"
             f"RULES for the open email:\n"
-            f"1. DRAFT a reply (default for any 'write/send/reply/tell them' "
+            f"1. DRAFT a reply (default for any 'write/reply/tell them' "
             f"request without a different recipient): call `ui_control` with "
-            f"`action=\"open_email_reply\"` and `extra=\"{_em_uid} {_em_folder} "
-            f"reply\"`. This opens the proper reply doc with To/Subject/"
+            f"`action=\"open_email_reply\"`, `uid=\"{_em_uid}\"`, "
+            f"`folder=\"{_em_folder}\"`, `mode=\"reply\"`, and `body` set to "
+            f"the reply text you wrote. This opens the proper reply doc with To/Subject/"
             f"In-Reply-To pre-filled by the backend. The user will see and edit "
             f"it before sending. DO NOT `create_document` a markdown file with "
             f"hand-written `To:` / `Subject:` / `In-Reply-To:` headers — that "
@@ -1350,9 +1617,9 @@ def _build_system_prompt(
             from src.settings import load_settings as _load_settings
             _style = (_load_settings().get("email_writing_style", "") or "").strip()
             if _style:
+                # Hardcoded identity/style rules stay in the trusted system prompt.
                 agent_prompt += (
-                    "\n\n📧 EMAIL WRITING STYLE AND IDENTITY — FOLLOW FOR ANY EMAIL DRAFT OR SEND:\n"
-                    f"{_style}\n\n"
+                    "\n\n"
                     "Hard identity rule: write as the user/mailbox owner only. Do not sign as, speak as, "
                     "or imply you are the recipient, original sender, quoted sender, spouse, assistant, "
                     "company, or any other third party. If a signature is needed, use only the name/signature "
@@ -1361,6 +1628,12 @@ def _build_system_prompt(
                     "For English emails, default to Hi [Name] or Hiya from the saved style rather than Hey. "
                     "If the saved style specifies Best/newline/name, use that sign-off when a sign-off is natural."
                 )
+                # User-editable style text is untrusted — wrap it so a malicious
+                # style value cannot inject system-role instructions.
+                _email_style_message = untrusted_context_message(
+                    "email writing style",
+                    "EMAIL WRITING STYLE AND IDENTITY — FOLLOW FOR ANY EMAIL DRAFT OR SEND:\n" + _style,
+                )
         except Exception:
             pass
 
@@ -1488,6 +1761,25 @@ def _build_system_prompt(
         except Exception as _sk_err:
             logger.debug(f"skill injection failed (non-fatal): {_sk_err}")
 
+    # Integration descriptions — user-editable fields, must not be in system role.
+    if not suppress_local_context:
+        try:
+            from src.integrations import get_integrations_prompt
+            _integ_prompt = get_integrations_prompt()
+            if _integ_prompt:
+                _integ_message = untrusted_context_message("integrations", _integ_prompt)
+        except Exception as _integ_err:
+            logger.debug(f"Integration prompt injection skipped: {_integ_err}")
+
+    # MCP tool descriptions — sourced from external servers, must not be in system role.
+    if mcp_mgr:
+        try:
+            _mcp_desc = mcp_mgr.get_tool_descriptions_for_prompt(mcp_disabled_map or {})
+            if _mcp_desc:
+                _mcp_desc_message = untrusted_context_message("MCP tools", _mcp_desc)
+        except Exception as _mcp_err:
+            logger.debug(f"MCP description injection skipped: {_mcp_err}")
+
     agent_msg = {"role": "system", "content": agent_prompt}
     insert_idx = 0
     for i, msg in enumerate(messages):
@@ -1527,6 +1819,15 @@ def _build_system_prompt(
     if _email_message:
         merged.insert(last_user_idx, _email_message)
         last_user_idx += 1
+    if _email_style_message:
+        merged.insert(last_user_idx, _email_style_message)
+        last_user_idx += 1
+    if _integ_message:
+        merged.insert(last_user_idx, _integ_message)
+        last_user_idx += 1
+    if _mcp_desc_message:
+        merged.insert(last_user_idx, _mcp_desc_message)
+        last_user_idx += 1
     if _skills_message:
         merged.insert(last_user_idx, _skills_message)
         last_user_idx += 1
@@ -1633,24 +1934,17 @@ def _build_base_prompt(
             # Skill index is a soft enhancement — never fail prompt assembly on it.
             logger.debug(f"Skill-index injection skipped: {_e}")
 
-    # Inject integration descriptions
-    if not suppress_local_context:
-        from src.integrations import get_integrations_prompt
-        integ_prompt = get_integrations_prompt()
-        if integ_prompt:
-            agent_prompt += "\n\n" + integ_prompt
-
-    # Inject MCP tool descriptions
-    if mcp_mgr:
-        mcp_desc = mcp_mgr.get_tool_descriptions_for_prompt(mcp_disabled_map or {})
-        if mcp_desc:
-            agent_prompt += mcp_desc
-
     return agent_prompt, skill_index_block
 
 
 
-def _resolve_tool_blocks(round_response: str, native_tool_calls: list, round_num: int, is_api_model: bool = False):
+def _resolve_tool_blocks(
+    round_response: str,
+    native_tool_calls: list,
+    round_num: int,
+    is_api_model: bool = False,
+    allow_fenced_for_api: bool = False,
+):
     """Choose native function calls or fenced code block parsing. Returns (tool_blocks, used_native)."""
     used_native = False
     converted_calls = []  # native calls that converted, ALIGNED with tool_blocks
@@ -1683,7 +1977,7 @@ def _resolve_tool_blocks(round_response: str, native_tool_calls: list, round_num
         # falling back to DSML). Dropping the whole parser would silently lose
         # those too. Non-native / textual-only models keep every pattern,
         # fenced blocks included, since that's their *only* tool channel.
-        tool_blocks = parse_tool_blocks(round_response, skip_fenced=is_api_model)
+        tool_blocks = parse_tool_blocks(round_response, skip_fenced=(is_api_model and not allow_fenced_for_api))
         if tool_blocks:
             logger.info(f"Agent round {round_num}: {len(tool_blocks)} fenced tool block(s) detected")
 
@@ -2080,13 +2374,15 @@ async def stream_agent_loop(
     _intent = _classify_agent_request(messages, _last_user)
     _low_signal_turn = bool(_intent.get("low_signal"))
     _casual_low_signal_turn = _is_casual_low_signal(_last_user)
+    _active_document_relevant = _turn_targets_active_document(_intent, _last_user, active_document)
+    _prompt_active_document = active_document if _active_document_relevant else None
     _direct_low_signal = (
         _low_signal_turn
         and not bool(_intent.get("continuation"))
         and not plan_mode
         and not approved_plan
         and not guide_only
-        and (_casual_low_signal_turn or active_document is None)
+        and (_casual_low_signal_turn or not _active_document_relevant)
         and (_casual_low_signal_turn or not active_email)
         and (_casual_low_signal_turn or not workspace)
         and not forced_tools
@@ -2096,11 +2392,12 @@ async def stream_agent_loop(
     # user turns only for explicit continuations ("yes", "do it", "1").
     _retrieval_query = str(_intent.get("retrieval_query") or _last_user)
     logger.info(
-        "[agent-intent] latest=%r continuation=%s low_signal=%s domains=%s retrieval_query=%r",
+        "[agent-intent] latest=%r continuation=%s low_signal=%s domains=%s active_doc_relevant=%s retrieval_query=%r",
         _last_user[:120],
         bool(_intent.get("continuation")),
         _low_signal_turn,
         sorted(_intent.get("domains") or []),
+        _active_document_relevant,
         _retrieval_query[:200],
     )
     _mcp_disabled_map = _load_mcp_disabled_map() if mcp_mgr else {}
@@ -2279,10 +2576,11 @@ async def stream_agent_loop(
         if "ui" in (_intent.get("domains") or set()):
             _relevant_tools.add("ui_control")
 
-    # If a document is open the model needs the editing tools available
-    # regardless of which selection path (RAG, keyword, caller-provided) ran
-    # or what keywords were in the latest user message.
-    if _relevant_tools is not None and active_document is not None:
+    # If this turn targets the open document, keep editing tools available
+    # regardless of which selection path (RAG, keyword, caller-provided) ran.
+    # Do not leak document tools into unrelated turns just because the editor
+    # panel is open.
+    if _relevant_tools is not None and _active_document_relevant:
         _relevant_tools.update({"edit_document", "update_document", "suggest_document"})
 
     # Current-turn chat uploads are real files under the upload/data root. Make
@@ -2341,6 +2639,28 @@ async def stream_agent_loop(
         except Exception as _e:
             logger.debug(f"[tool-rag] skill-aware tool include skipped: {_e}")
 
+    _intent_domains = set(_intent.get("domains") or set())
+    _ody_doc_finetune_mode = (
+        (model or "").lower().startswith("odysseus-qwen3")
+        and (
+            "documents" in _intent_domains
+            or _active_document_relevant
+            or _prompt_active_document is not None
+        )
+        and "files" not in _intent_domains
+        and not guide_only
+    )
+    _ody_doc_stream_create_mode = _ody_doc_finetune_mode and _prompt_active_document is None
+    if _ody_doc_finetune_mode and _relevant_tools is not None:
+        if _prompt_active_document is not None:
+            _relevant_tools = {
+                "edit_document", "update_document", "suggest_document",
+                "ask_user", "update_plan",
+            }
+        else:
+            _relevant_tools = {"create_document", "ask_user", "update_plan"}
+        logger.info("[agent-intent] odysseus doc finetune tool clamp=%s", sorted(_relevant_tools))
+
     if _relevant_tools is not None:
         logger.info("[agent-intent] selected_tools=%s", sorted(_relevant_tools)[:50])
 
@@ -2419,7 +2739,7 @@ async def stream_agent_loop(
         _is_api_model = any(h in endpoint_url for h in _API_HOSTS) or _model_supports_tools
     _compact_agent_prompt = _is_api_model or _is_ollama_native or _ollama_openai_compat
     messages, mcp_schemas = _build_system_prompt(
-        messages, model, active_document, mcp_mgr, disabled_tools,
+        messages, model, _prompt_active_document, mcp_mgr, disabled_tools,
         needs_admin=_needs_admin, relevant_tools=_relevant_tools,
         mcp_disabled_map=_mcp_disabled_map,
         compact=_compact_agent_prompt,
@@ -2428,6 +2748,19 @@ async def stream_agent_loop(
         suppress_skills=_low_signal_turn,
         active_email=active_email,
     )
+    if _ody_doc_finetune_mode and not plan_mode and not approved_plan and not guide_only:
+        messages = _minimal_odysseus_doc_messages(
+            messages,
+            _prompt_active_document,
+            stream_create=_ody_doc_stream_create_mode,
+        )
+        mcp_schemas = []
+        logger.info(
+            "[agent-intent] odysseus doc minimal prompt active active_doc=%s stream_create=%s messages=%s",
+            bool(_prompt_active_document),
+            _ody_doc_stream_create_mode,
+            len(messages),
+        )
     if plan_mode and not guide_only:
         # Steer the model to investigate-then-propose. Hard tool gating handles
         # every write path except shell; this directive is what keeps the
@@ -2583,6 +2916,8 @@ async def stream_agent_loop(
     _doc_acc = ""          # accumulated tool-call JSON arguments
     _doc_opened = False    # whether doc_stream_open was sent
     _doc_last_len = 0      # last content length sent
+    _doc_stream_create_completed = False
+    _ody_doc_tool_completed = False
 
     # Set when the loop runs out of rounds while the agent was still actively
     # using tools — i.e. it was cut off, not finished. Drives a "Continue" event
@@ -2637,6 +2972,8 @@ async def stream_agent_loop(
                     if s.get("function", {}).get("name") not in _ADMIN_SCHEMA_NAMES
                 ]
                 all_tool_schemas = base_schemas + mcp_schemas
+            if _ody_doc_finetune_mode:
+                all_tool_schemas = []
             if disabled_tools:
                 all_tool_schemas = [
                     t for t in all_tool_schemas
@@ -2682,6 +3019,7 @@ async def stream_agent_loop(
             max_tokens=max_tokens,
             prompt_type=prompt_type if round_num == 1 else None,
             tools=all_tool_schemas if all_tool_schemas else None,
+            tool_choice_none=_ody_doc_finetune_mode,
             timeout=agent_stream_timeout,
             session_id=session_id,
         ):
@@ -2805,17 +3143,32 @@ async def stream_agent_loop(
                         if data.get("thinking"):
                             round_reasoning += data["delta"]
                         else:
-                            round_response += data["delta"]
-                            full_response += data["delta"]
-                        yield chunk  # Stream all rounds
-                        # Detect text-fence doc streaming for rounds 2+
-                        # (round 1 is handled by frontend fence detection + server fenced block path)
+                            _delta_text = _strip_doc_model_artifacts(data["delta"]) if _ody_doc_finetune_mode else data["delta"]
+                            round_response += _delta_text
+                            full_response += _delta_text
+                            data["delta"] = _delta_text
+                        if not _ody_doc_finetune_mode or data.get("thinking"):
+                            yield f"data: {json.dumps(data)}\n\n"
+                        # Detect text-fence doc streaming. Normal agent prompts
+                        # use ```create_document; the doc LoRA streaming path
+                        # uses neutral ```document to avoid triggering learned
+                        # hidden native tool-call output.
                         if (
-                            round_num > 1
+                            (round_num > 1 or _ody_doc_stream_create_mode)
                             and not _doc_acc
                             and not (tool_policy and tool_policy.blocks("create_document"))
                         ):
-                            _fence_marker = '```create_document\n'
+                            _fence_markers = (
+                                ('```document\n', '```documen\n')
+                                if _ody_doc_stream_create_mode
+                                else ('```create_document\n',)
+                            )
+                            _fence_marker = None
+                            for _mk in _fence_markers:
+                                _candidate = _mk[0] if isinstance(_mk, tuple) else _mk
+                                if _candidate in round_response[_doc_scan_from:]:
+                                    _fence_marker = _candidate
+                                    break
                             # Open a new block if we're not currently inside one
                             # and there's an unstreamed marker in the response.
                             # The marker search starts at the byte after the
@@ -2823,7 +3176,7 @@ async def stream_agent_loop(
                             # `create_document` block in the same round gets
                             # detected (previously only the first one was
                             # streamed and the rest were silently dropped).
-                            if not _doc_opened and _fence_marker in round_response[_doc_scan_from:]:
+                            if not _doc_opened and _fence_marker:
                                 _fi = round_response.index(_fence_marker, _doc_scan_from)
                                 _fa = round_response[_fi + len(_fence_marker):]
                                 _fl = _fa.split('\n')
@@ -2875,12 +3228,45 @@ async def stream_agent_loop(
             _round_first_event_logged,
             _round_first_token_logged,
         )
+        _normalized_doc_round = (
+            _normalize_stream_document_fences(
+                round_response,
+                "create_document" if _ody_doc_stream_create_mode else "update_document",
+            )
+            if _ody_doc_finetune_mode
+            else round_response
+        )
         tool_blocks, used_native, converted_calls = _resolve_tool_blocks(
-            round_response,
+            _normalized_doc_round,
             native_tool_calls,
             round_num,
             is_api_model=(_is_api_model and not guide_only),
+            allow_fenced_for_api=_ody_doc_finetune_mode,
         )
+        if _ody_doc_stream_create_mode and tool_blocks:
+            create_idx = next(
+                (idx for idx, block in enumerate(tool_blocks) if block.tool_type == "create_document"),
+                None,
+            )
+            if create_idx is None:
+                logger.info(
+                    "[agent] odysseus doc stream-create discarded non-create tool call(s): %s",
+                    [block.tool_type for block in tool_blocks],
+                )
+                tool_blocks = []
+                converted_calls = []
+            else:
+                if len(tool_blocks) > 1 or create_idx != 0:
+                    logger.info(
+                        "[agent] odysseus doc stream-create keeping first create_document and dropping extras: %s",
+                        [block.tool_type for block in tool_blocks],
+                    )
+                tool_blocks = [tool_blocks[create_idx]]
+                converted_calls = (
+                    [converted_calls[create_idx]]
+                    if create_idx < len(converted_calls)
+                    else converted_calls[:1]
+                )
 
         # Force-answer round: we told the model to STOP calling tools and
         # answer. If it ignored that and emitted a (possibly DSML) tool
@@ -3171,10 +3557,11 @@ async def stream_agent_loop(
             # Build a short display string for the frontend tool bubble.
             # Document tools show a brief summary instead of dumping full content.
             is_doc_tool = block.tool_type in ("create_document", "update_document", "edit_document", "suggest_document")
+            full_command = block.content.strip()
             if is_doc_tool:
                 cmd_display = block.content.split("\n")[0].strip()[:80]
             else:
-                cmd_display = block.content.strip()
+                cmd_display = full_command
 
             if tool_policy and tool_policy.blocks(block.tool_type):
                 desc = f"{block.tool_type}: BLOCKED"
@@ -3186,7 +3573,7 @@ async def stream_agent_loop(
                 logger.info("Tool blocked before start by policy: %s", block.tool_type)
             else:
                 yield (
-                    f'data: {json.dumps({"type": "tool_start", "tool": block.tool_type, "command": cmd_display, "round": round_num})}\n\n'
+                    f'data: {json.dumps({"type": "tool_start", "tool": block.tool_type, "command": cmd_display, "full_command": full_command, "round": round_num})}\n\n'
                 )
 
                 # Streaming progress for long-running tools (bash, python).
@@ -3383,6 +3770,15 @@ async def stream_agent_loop(
 
             # Emit tool_output (include ui_event data if present)
             tool_output_data = {"type": "tool_output", "tool": block.tool_type, "command": cmd_display, "output": output_text, "exit_code": result.get("exit_code")}
+            if is_doc_tool and "action" in result:
+                tool_output_data.update({
+                    "doc_id": result.get("doc_id"),
+                    "document_action": result.get("action"),
+                    "document_title": result.get("title", ""),
+                    "document_language": result.get("language", ""),
+                    "document_version": result.get("version"),
+                    "document_content": result.get("content", ""),
+                })
             if _pending_ask_user_event:
                 # Keep enough state in the streamed tool result for alternate
                 # clients to render the prompt without depending on event order.
@@ -3494,6 +3890,18 @@ async def stream_agent_loop(
             formatted = format_tool_result(desc, result)
             tool_results.append(formatted)
             tool_result_texts.append(formatted)
+            if (
+                _ody_doc_stream_create_mode
+                and block.tool_type == "create_document"
+                and result.get("action") == "create"
+            ):
+                _doc_stream_create_completed = True
+            if (
+                _ody_doc_finetune_mode
+                and block.tool_type in ("create_document", "update_document", "edit_document", "suggest_document")
+                and not result.get("error")
+            ):
+                _ody_doc_tool_completed = True
 
         # If budget was hit, stop the loop
         if budget_hit:
@@ -3506,6 +3914,20 @@ async def stream_agent_loop(
         if _awaiting_user:
             break
 
+        if _doc_stream_create_completed:
+            if not full_response.strip():
+                full_response = "Done."
+                yield 'data: ' + json.dumps({"delta": "Done."}) + '\n\n'
+            logger.info("[agent] odysseus doc stream-create completed after one create_document")
+            break
+
+        if _ody_doc_tool_completed:
+            if not full_response.strip() or full_response.strip().startswith("```"):
+                full_response = "Done."
+                yield 'data: ' + json.dumps({"delta": "Done."}) + '\n\n'
+            logger.info("[agent] odysseus doc tool completed after one textual tool block")
+            break
+
         # Feed results back to LLM for next round
         # Pass the CONVERTED calls (aligned 1:1 with tool_result_texts), not the
         # raw native_tool_calls: a call that failed to convert is dropped from

src/agent_tools/__init__.py

@@ -14,6 +14,7 @@ Sub-modules:
 import logging
 from collections import namedtuple
 
+from src.tool_security import BUILTIN_EMAIL_TOOLS
 from src.tool_utils import _truncate, get_mcp_manager, set_mcp_manager
 
 logger = logging.getLogger(__name__)
@@ -86,9 +87,10 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi
              "manage_endpoints", "manage_mcp", "manage_webhooks",
              "manage_tokens", "manage_documents", "manage_settings",
              "manage_notes", "manage_calendar",
-             "resolve_contact", "manage_contact", "list_email_accounts", "send_email", "list_emails",
-             "read_email", "reply_to_email", "bulk_email", "archive_email",
-             "delete_email", "mark_email_read",
+             "resolve_contact", "manage_contact",
+             # Email tool names come from BUILTIN_EMAIL_TOOLS (unioned below)
+             # so the fence regex, dispatch, and non-admin blocklist all cover
+             # the same set.
              # Cookbook tools (LLM serving + downloads). Without these
              # entries, native function calls to e.g. list_served_models
              # are rejected as "Unknown function call" before reaching
@@ -105,7 +107,7 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi
              # Generic loopback to any UI-button endpoint (cookbook,
              # gallery, email folders, etc.) — agent uses this when
              # there's no named tool wrapper for the action.
-             "app_api"}
+             "app_api"} | BUILTIN_EMAIL_TOOLS
 
 ToolBlock = namedtuple("ToolBlock", ["tool_type", "content"])
 

src/agent_tools/admin_tools.py

@@ -14,6 +14,7 @@ import logging
 from typing import Optional, Dict
 
 from src.tool_utils import get_mcp_manager, _parse_tool_args
+from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
@@ -706,7 +707,14 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
                 "tasks": ["manage_tasks"],
                 "notes": ["manage_notes"],
                 "calendar": ["manage_calendar"],
-                "email": ["mcp__email__list_emails", "mcp__email__read_email", "mcp__email__send_email"],
+                # The full built-in email tool set, in BOTH spellings: the
+                # qualified mcp__email__* names drive MCP schema hiding, the
+                # bare names drive function-schema hiding, and the runtime
+                # gate accepts either — deriving from BUILTIN_EMAIL_TOOLS
+                # keeps the toggle covering every tool the email server
+                # exposes instead of a hand-picked subset.
+                "email": sorted(BUILTIN_EMAIL_TOOLS)
+                         + [f"mcp__email__{t}" for t in sorted(BUILTIN_EMAIL_TOOLS)],
                 "research": ["web_search", "web_fetch"],  # research is a per-request flag, not a tool (closest analog)
             }
 

src/agent_tools/document_tools.py

@@ -185,6 +185,71 @@ def parse_suggest_blocks(content: str) -> list:
     return suggestions
 
 
+def _pdf_source_upload_id(content: str) -> Optional[str]:
+    try:
+        from src.pdf_form_doc import find_source_upload_id
+        return find_source_upload_id(content or "")
+    except Exception:
+        return None
+
+
+def _strip_pdf_editor_markers(content: str) -> str:
+    """Turn a PDF-wrapper markdown doc into ordinary editable markdown.
+
+    PDF docs use hidden HTML comments for source-upload links, form fields, and
+    page annotations. Those comments are necessary for rendering/exporting the
+    original PDF, but they make a derived AI text edit keep showing the original
+    PDF preview. Remove only the editor plumbing and keep the readable text.
+    """
+    text = content or ""
+    text = re.sub(r'(?im)^\s*<!--\s*pdf(?:_form)?_source\s+[^>]*-->\s*\n*', '', text)
+    text = re.sub(r'\s*<!--\s*field=[^>]*-->', '', text)
+    text = re.sub(r'\s*<!--\s*annotation\s+[^>]*-->', '', text)
+    return text.strip()
+
+
+def _create_pdf_text_derivative(db, *, source_doc, content: str, owner: Optional[str], summary: str) -> dict:
+    import uuid
+    from src.database import Document, DocumentVersion
+
+    clean = _strip_pdf_editor_markers(content)
+    title_base = (getattr(source_doc, "title", None) or "PDF").strip()
+    title = title_base if title_base.lower().endswith("edited") else f"{title_base} edited"
+    doc_id = str(uuid.uuid4())
+    ver_id = str(uuid.uuid4())
+    new_doc = Document(
+        id=doc_id,
+        session_id=getattr(source_doc, "session_id", None),
+        title=title,
+        language="markdown",
+        current_content=clean,
+        version_count=1,
+        is_active=True,
+        owner=owner if owner is not None else getattr(source_doc, "owner", None),
+    )
+    ver = DocumentVersion(
+        id=ver_id,
+        document_id=doc_id,
+        version_number=1,
+        content=clean,
+        summary=summary,
+        source="ai",
+    )
+    db.add(new_doc)
+    db.add(ver)
+    db.commit()
+    set_active_document(doc_id)
+    return {
+        "action": "create",
+        "doc_id": doc_id,
+        "title": title,
+        "language": "markdown",
+        "content": clean,
+        "version": 1,
+        "source_doc_id": getattr(source_doc, "id", None),
+    }
+
+
 class CreateDocumentTool:
     async def execute(self, content: str, ctx: dict) -> dict:
         """Create a new document. Supports two formats:
@@ -332,6 +397,15 @@ class UpdateDocumentTool:
             if is_email_doc:
                 doc.language = "email"
 
+            if not is_email_doc and _pdf_source_upload_id(doc.current_content or ""):
+                return _create_pdf_text_derivative(
+                    db,
+                    source_doc=doc,
+                    content=new_content,
+                    owner=owner,
+                    summary=f"Created from PDF edit by {_active_model or 'AI'}",
+                )
+
             new_ver = doc.version_count + 1
             ver = DocumentVersion(
                 id=str(uuid.uuid4()),
@@ -416,6 +490,15 @@ class EditDocumentTool:
             if applied == 0:
                 return {"error": f"No edits applied — none of the FIND blocks matched the document content (skipped {skipped})"}
 
+            if _pdf_source_upload_id(doc.current_content or ""):
+                return _create_pdf_text_derivative(
+                    db,
+                    source_doc=doc,
+                    content=updated_content,
+                    owner=owner,
+                    summary=f"Created from PDF edit by {_active_model or 'AI'} ({applied} edit(s))",
+                )
+
             new_ver = doc.version_count + 1
             ver = DocumentVersion(
                 id=str(uuid.uuid4()),

src/agent_tools/filesystem_tools.py

@@ -186,6 +186,21 @@ class WriteFileTool:
         lines = content.split("\n", 1)
         raw_path = lines[0].strip()
         body = lines[1] if len(lines) > 1 else ""
+        # Decode JSON-object args (the fenced inline-args shape
+        # ```write_file {"path": "...", "content": "..."}```), matching
+        # ReadFileTool above. Without this the whole JSON string becomes the
+        # path and the file is written under a garbage name. This is the live
+        # path: there is no filesystem MCP server, so write_file always runs
+        # here via _direct_fallback, not through _build_mcp_args.
+        _stripped = content.strip()
+        if _stripped.startswith("{"):
+            try:
+                _a = json.loads(_stripped)
+                if isinstance(_a, dict) and "path" in _a:
+                    raw_path = str(_a.get("path", "")).strip()
+                    body = str(_a.get("content", ""))
+            except (json.JSONDecodeError, TypeError, ValueError):
+                pass
         try:
             path = _resolve_tool_path(raw_path)
         except ValueError as e:
@@ -266,7 +281,13 @@ class LsTool:
 
 class GlobTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
+        from src.tool_execution import (
+            _SENSITIVE_BASENAMES,
+            _is_sensitive_path,
+            _resolve_tool_path,
+            _resolve_search_root,
+            _truncate,
+        )
         args = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -288,11 +309,30 @@ class GlobTool:
             base = os.path.abspath(root)
             if not os.path.isdir(base):
                 return None, f"glob: {root}: not a directory"
+            rbase = os.path.realpath(base)
             norm_pat = pattern.replace("\\", "/")
             # Fast path: literal pattern (no wildcards) → direct path lookup.
             if not any(c in norm_pat for c in "*?["):
-                cand = os.path.normpath(os.path.join(base, norm_pat))
-                if os.path.exists(cand):
+                cand = os.path.realpath(os.path.join(base, norm_pat))
+                # Keep the literal lookup inside the search root. os.path.join
+                # lets an absolute pattern (or one containing ../) escape `base`,
+                # which would turn glob into an existence/path oracle for
+                # arbitrary host files — bypassing the workspace/allowlist
+                # confinement that _resolve_search_root applies to the root.
+                # An escaping literal falls through to the walk, which only ever
+                # yields paths under base.
+                nbase = os.path.normcase(rbase)
+                try:
+                    inside = cand == rbase or os.path.commonpath(
+                        [os.path.normcase(cand), nbase]
+                    ) == nbase
+                except ValueError:
+                    inside = False
+                # A literal that names a deny-listed sensitive file (.env,
+                # .ssh/id_rsa, …) falls through to the walk, which skips it —
+                # otherwise glob would surface secret paths that read_file /
+                # grep already refuse to touch.
+                if inside and os.path.exists(cand) and not _is_sensitive_path(cand):
                     return [cand], None
                 # Literal not at exact path — fall through to walk so
                 # e.g. "foo.py" still matches at any depth (like rglob).
@@ -304,11 +344,20 @@ class GlobTool:
                 for dp, dns, fns in os.walk(base):
                     # Prune skipped dirs before descending (unlike rglob which
                     # descends first then filters — fatal on large node_modules).
-                    dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
+                    # Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
+                    # never enumerates the keys/tokens inside them.
+                    dns[:] = [
+                        d for d in dns
+                        if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
+                    ]
                     for name in fns + dns:
                         full = os.path.join(dp, name)
                         rel = os.path.relpath(full, base).replace(os.sep, "/")
                         if regex.fullmatch(rel) or regex.fullmatch(name):
+                            # Skip deny-listed sensitive files (.env, id_rsa,
+                            # known_hosts, …) the same way grep does.
+                            if _is_sensitive_path(os.path.realpath(full)):
+                                continue
                             try:
                                 mtime = os.stat(full).st_mtime
                             except OSError:
@@ -333,7 +382,13 @@ class GlobTool:
 
 class GrepTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
+        from src.tool_execution import (
+            _SENSITIVE_FILE_PATTERNS,
+            _is_sensitive_path,
+            _resolve_tool_path,
+            _resolve_search_root,
+            _truncate,
+        )
         args: Dict[str, Any] = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -369,6 +424,8 @@ class GrepTool:
                     cmd.append("--ignore-case")
                 if glob_pat:
                     cmd += ["--glob", glob_pat]
+                for _pat in _SENSITIVE_FILE_PATTERNS:
+                    cmd += ["--glob", f"!*{_pat}*"]
                 for _d in _CODENAV_SKIP_DIRS:
                     cmd += ["--glob", f"!**/{_d}/**"]
                 cmd += ["--regexp", pattern, root]
@@ -399,6 +456,8 @@ class GrepTool:
             for fp in file_iter:
                 if len(hits) >= max_hits:
                     break
+                if _is_sensitive_path(os.path.realpath(fp)):
+                    continue
                 try:
                     with open(fp, "r", encoding="utf-8", errors="strict") as f:
                         for i, line in enumerate(f, 1):

src/agent_tools/interaction_tools.py

@@ -18,7 +18,7 @@ class AskUserTool:
             parsed = json.loads(raw) if raw else {}
         except (ValueError, TypeError):
             parsed = {}
-            
+
         if isinstance(parsed, dict):
             question = str(parsed.get("question", "")).strip()
             multi = bool(parsed.get("multi") or parsed.get("multiSelect"))
@@ -34,7 +34,7 @@ class AskUserTool:
                     options.append({"label": label, "description": descr})
         else:
             question = raw
-            
+
         if not question or len(options) < 2:
             return "ask_user: invalid", {
                 "error": (
@@ -43,7 +43,7 @@ class AskUserTool:
                 ),
                 "exit_code": 1,
             }
-            
+
         options = options[:6]  # keep the choice list sane
         desc = f"ask_user: {question[:80]}"
         labels = ", ".join(o["label"] for o in options)
@@ -70,18 +70,18 @@ class UpdatePlanTool:
             parsed = json.loads(raw) if raw else {}
         except (ValueError, TypeError):
             parsed = {}
-            
+
         if isinstance(parsed, dict) and parsed.get("plan"):
             plan = str(parsed.get("plan", "")).strip()
         else:
             plan = raw
-            
+
         if not plan:
             return "update_plan: invalid", {
                 "error": "update_plan needs a non-empty `plan` (the full updated checklist as markdown).",
                 "exit_code": 1,
             }
-            
+
         plan = plan[:8192]
         done = plan.count("- [x]") + plan.count("- [X]")
         total = done + plan.count("- [ ]")

src/agent_tools/session_tools.py

@@ -104,6 +104,8 @@ async def list_sessions(content: str, session_id: Optional[str] = None, owner: O
         sessions = _session_manager.get_sessions_for_user(owner)
         rows = []
         for sid, sess in sessions.items():
+            if (sess.name or "").startswith("SFT trace batch"):
+                continue
             if keyword and keyword not in (sess.name or "").lower():
                 continue
             db_row = db_rows.get(sid)
@@ -192,6 +194,25 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner:
     try:
         # Build context from session history
         context = sess.get_context_messages()
+        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
+        model = str(getattr(sess, "model", "") or "")
+        if model == "fixture-tool-model" or "host.docker.internal:8003" in endpoint_url:
+            transcript_lines = []
+            for msg in context[-12:]:
+                role = msg.get("role", "unknown")
+                text = (msg.get("content") or "").strip()
+                if text:
+                    transcript_lines.append(f"{role}: {text}")
+            transcript = "\n".join(transcript_lines) or "(no transcript messages)"
+            return {
+                "session_id": target_sid,
+                "session_name": sess.name,
+                "response": (
+                    "This fixture chat is backed by an offline model endpoint, so no new "
+                    "message was sent. Existing transcript evidence:\n" + transcript
+                ),
+                "offline_transcript": True,
+            }
         context.append({"role": "user", "content": message})
 
         response = await llm_call_async(

src/ai_interaction.py

@@ -433,13 +433,23 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
             return {"error": "Search needs line 2: query"}
         query = lines[1].strip()
         memories = _memory_manager.load(owner=owner)
+        query_lower = query.lower()
+        exact_results = [m for m in memories if query_lower in (m.get("text", "").lower())]
 
         if hasattr(_memory_manager, 'get_relevant_memories'):
-            results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+            vector_results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
         else:
-            # Fallback: simple text search
-            query_lower = query.lower()
-            results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
+            vector_results = []
+        seen = set()
+        results = []
+        for m in [*exact_results, *vector_results]:
+            mid = m.get("id")
+            if mid in seen:
+                continue
+            seen.add(mid)
+            results.append(m)
+            if len(results) >= 20:
+                break
 
         if not results:
             return {"results": f"No memories found matching '{query}'."}

src/builtin_actions.py

@@ -7,6 +7,7 @@ scheduler without needing an LLM call.
 
 import logging
 import os
+import json
 from datetime import datetime
 from typing import Tuple
 
@@ -14,6 +15,7 @@ from src.auth_helpers import owner_filter
 from core.platform_compat import IS_WINDOWS, find_bash
 from core.constants import internal_api_base
 from src.constants import DATA_DIR, DEEP_RESEARCH_DIR, TIDY_CALENDAR_STATE_FILE, EMAIL_URGENCY_CACHE_DIR, COOKBOOK_STATE_FILE
+from src.interactive_gate import wait_for_interactive_quiet
 
 logger = logging.getLogger(__name__)
 
@@ -145,6 +147,7 @@ async def action_consolidate_memory(owner: str, **kwargs) -> Tuple[str, bool]:
                     "\"drop\":[{\"id\":\"existing id\",\"reason\":\"short reason\"}]}\n\n"
                     f"MEMORIES:\n{json.dumps(items, ensure_ascii=False)}"
                 )
+                await wait_for_interactive_quiet("memory consolidation action")
                 raw = await llm_call_async_with_fallback(
                     candidates,
                     messages=[{"role": "user", "content": prompt}],
@@ -497,11 +500,48 @@ def _result_has_work(result: str | None) -> bool:
     return True
 
 
+def _result_is_config_error(result: str | None) -> bool:
+    if not isinstance(result, str):
+        return False
+    low = result.lower()
+    return (
+        "no model configured" in low
+        or "no model endpoint configured" in low
+        or "no llm endpoint available" in low
+    )
+
+
+def _email_task_account_id(kwargs) -> str | None:
+    prompt = (kwargs.get("prompt") or "").strip()
+    if not prompt:
+        return None
+    try:
+        data = json.loads(prompt)
+        if isinstance(data, dict):
+            val = data.get("account_id") or data.get("email_account_id")
+            return str(val).strip() or None
+    except Exception:
+        pass
+    for line in prompt.splitlines():
+        if "=" not in line:
+            continue
+        key, val = line.split("=", 1)
+        if key.strip().lower() in {"account_id", "email_account_id"}:
+            return val.strip() or None
+    return None
+
+
 async def action_summarize_emails(owner: str, **kwargs) -> Tuple[str, bool]:
     """Run one pass of email summary background processing."""
     try:
         from routes.email_pollers import _run_auto_summarize_once
-        result = await _run_auto_summarize_once(do_summary=True, do_reply=False)
+        result = await _run_auto_summarize_once(
+            do_summary=True,
+            do_reply=False,
+            account_id=_email_task_account_id(kwargs),
+        )
+        if _result_is_config_error(result):
+            return result, False
         if not _result_has_work(result):
             raise TaskNoop(f"summarize: {result or 'no new emails'}")
         return result, True
@@ -517,9 +557,12 @@ async def action_draft_email_replies(owner: str, **kwargs) -> Tuple[str, bool]:
         result = await _run_auto_summarize_once(
             do_summary=False,
             do_reply=True,
+            account_id=_email_task_account_id(kwargs),
             days_back=7,
             progress_cb=kwargs.get("progress_cb"),
         )
+        if _result_is_config_error(result):
+            return result, False
         if not _result_has_work(result):
             raise TaskNoop(f"draft replies: {result or 'no new emails'}")
         return result, True
@@ -528,6 +571,250 @@ async def action_draft_email_replies(owner: str, **kwargs) -> Tuple[str, bool]:
         return str(e), False
 
 
+async def action_email_auto_translate(owner: str, **kwargs) -> Tuple[str, bool]:
+    """Detect recent foreign-language emails and cache translated text.
+
+    The reader still shows the original body; it simply checks this cache
+    before calling the LLM on demand. Keep the scheduled pass deliberately
+    small so translation never turns into a mailbox-wide background crawl.
+    """
+    try:
+        import email as _email_mod
+        import json as _json
+        import re as _re
+        import sqlite3 as _sql3
+        from datetime import datetime as _dt, timedelta as _td
+
+        from core.database import EmailAccount as _EA, SessionLocal as _SL
+        from routes.email_helpers import (
+            SCHEDULED_DB,
+            _decode_header,
+            _email_cache_owner_clause,
+            _extract_reply,
+            _extract_text,
+            _imap_connect,
+            email_translation_body_hash,
+        )
+        from src.settings import load_settings
+        from src.task_endpoint import task_llm_call_async
+
+        settings = load_settings()
+        if not settings.get("email_auto_translate", False):
+            raise TaskNoop("email auto-translate is disabled")
+
+        target_language = (settings.get("email_translate_language") or "English").strip() or "English"
+        account_id = _email_task_account_id(kwargs)
+        days_back = 7
+        max_process = 5
+        try:
+            data = _json.loads((kwargs.get("prompt") or "").strip() or "{}")
+            if isinstance(data, dict):
+                days_back = max(1, min(30, int(data.get("days_back") or days_back)))
+                max_process = max(1, min(20, int(data.get("max_process") or max_process)))
+        except Exception:
+            pass
+
+        db = _SL()
+        try:
+            from sqlalchemy import and_ as _and, or_ as _or
+            q = db.query(_EA).filter(_EA.enabled == True)  # noqa: E712
+            if owner:
+                unowned = _or(_EA.owner == None, _EA.owner == "")  # noqa: E711
+                same_mailbox = _or(_EA.imap_user == owner, _EA.from_address == owner)
+                q = q.filter(_or(_EA.owner == owner, _and(unowned, same_mailbox)))
+            if account_id:
+                q = q.filter(_EA.id == account_id)
+            accounts = q.all()
+        finally:
+            db.close()
+        if not accounts:
+            raise TaskNoop("no email accounts configured")
+
+        def _cached(body_hash: str) -> bool:
+            c = _sql3.connect(SCHEDULED_DB)
+            try:
+                owner_clause, owner_params = _email_cache_owner_clause(owner)
+                row = c.execute(
+                    f"SELECT 1 FROM email_translations "
+                    f"WHERE body_hash = ? AND target_language = ? AND {owner_clause} LIMIT 1",
+                    (body_hash, target_language, *owner_params),
+                ).fetchone()
+                return bool(row)
+            finally:
+                c.close()
+
+        def _store(
+            body_hash: str,
+            *,
+            uid: str,
+            folder: str,
+            subject: str,
+            sender: str,
+            translation: str,
+            same_language: bool,
+            model_used: str,
+        ) -> None:
+            c = _sql3.connect(SCHEDULED_DB)
+            try:
+                c.execute("""
+                    INSERT OR REPLACE INTO email_translations
+                    (body_hash, owner, target_language, uid, folder, subject, sender,
+                     translation, same_language, model_used, created_at)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """, (
+                    body_hash, owner, target_language, uid, folder, subject, sender,
+                    translation, 1 if same_language else 0, model_used, _dt.utcnow().isoformat(),
+                ))
+                c.commit()
+            finally:
+                c.close()
+
+        async def _translate(body: str, subject: str, sender: str) -> tuple[str, bool]:
+            content = await task_llm_call_async(
+                [
+                    {
+                        "role": "system",
+                        "content": (
+                            "You translate emails faithfully. Preserve meaning, names, dates, money, addresses, "
+                            "bullet structure, and tone. Do not summarize or answer the email. "
+                            "Output only the translation between <<<TRANSLATION>>> and <<<END>>>. "
+                            "If the email is already primarily in the target language, output exactly "
+                            "<<<SAME_LANGUAGE>>>."
+                        ),
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            f"Target language: {target_language}\n\n"
+                            f"From: {sender}\nSubject: {subject}\n\n{body[:16000]}\n\n"
+                            "Translate the email unless it is already primarily in the target language.\n"
+                            "Return only:\n<<<TRANSLATION>>>\ntranslated text\n<<<END>>>"
+                        ),
+                    },
+                ],
+                owner=owner,
+                temperature=0.2,
+                max_tokens=8192,
+                timeout=180,
+            )
+            content = (content or "").strip()
+            content = _extract_reply(content)
+            if "<<<SAME_LANGUAGE>>>" in content:
+                return "", True
+            marker = _re.search(r"<<<TRANSLATION>>>\s*(.*?)\s*<<<END>>>", content, _re.S | _re.I)
+            if marker:
+                content = marker.group(1).strip()
+            else:
+                content = _re.sub(r"^\s*<<<TRANSLATION>>>\s*", "", content, flags=_re.I).strip()
+                content = _re.sub(r"\s*<<<END>>>\s*$", "", content, flags=_re.I).strip()
+            return content, False
+
+        since = (_dt.utcnow() - _td(days=days_back)).strftime("%d-%b-%Y")
+        examined = 0
+        cached = 0
+        translated = 0
+        same_language = 0
+        skipped = 0
+        failures = 0
+        processed = 0
+
+        for acct in accounts:
+            if processed >= max_process:
+                break
+            imap = None
+            try:
+                imap = _imap_connect(acct.id, owner=owner)
+                imap.select("INBOX", readonly=True)
+                status, data = imap.uid("SEARCH", None, f'(SINCE {since})')
+                if status != "OK" or not data or not data[0]:
+                    continue
+                uids = list(reversed(data[0].split()))[:50]
+                for uid_b in uids:
+                    if processed >= max_process:
+                        break
+                    uid = uid_b.decode("utf-8", errors="ignore") if isinstance(uid_b, bytes) else str(uid_b)
+                    status, msg_data = imap.uid("FETCH", uid, "(RFC822)")
+                    if status != "OK" or not msg_data:
+                        continue
+                    raw = None
+                    for part in msg_data:
+                        if isinstance(part, tuple) and len(part) > 1:
+                            raw = part[1]
+                            break
+                    if not raw:
+                        continue
+                    msg = _email_mod.message_from_bytes(raw)
+                    subject = _decode_header(msg.get("Subject", ""))
+                    sender = _decode_header(msg.get("From", ""))
+                    body = (_extract_text(msg) or "").strip()
+                    examined += 1
+                    if len(body) < 80:
+                        skipped += 1
+                        continue
+                    body_hash = email_translation_body_hash(body)
+                    if _cached(body_hash):
+                        cached += 1
+                        continue
+                    translation, is_same_language = await _translate(body, subject, sender)
+                    if is_same_language:
+                        _store(
+                            body_hash,
+                            uid=uid,
+                            folder="INBOX",
+                            subject=subject,
+                            sender=sender,
+                            translation="",
+                            same_language=True,
+                            model_used="background-task",
+                        )
+                        same_language += 1
+                        processed += 1
+                        continue
+                    if not translation:
+                        failures += 1
+                        continue
+                    _store(
+                        body_hash,
+                        uid=uid,
+                        folder="INBOX",
+                        subject=subject,
+                        sender=sender,
+                        translation=translation,
+                        same_language=False,
+                        model_used="background-task",
+                    )
+                    translated += 1
+                    processed += 1
+            except Exception as acct_e:
+                failures += 1
+                logger.warning(f"email_auto_translate account scan failed for {getattr(acct, 'id', '?')}: {acct_e}")
+            finally:
+                if imap:
+                    try:
+                        imap.logout()
+                    except Exception:
+                        pass
+
+        if translated == 0 and same_language == 0:
+            result = (
+                f"no uncached foreign-language emails found "
+                f"(examined {examined}, cached {cached}, skipped {skipped}, failures {failures})"
+            )
+            if failures:
+                return f"Email Auto Translate failed: {result}", False
+            raise TaskNoop(result)
+        return (
+            f"Email Auto Translate cached {translated} translation(s), marked {same_language} same-language "
+            f"(examined {examined}, already cached {cached}, skipped {skipped}, failures {failures})",
+            True,
+        )
+    except TaskNoop:
+        raise
+    except Exception as e:
+        logger.error(f"email_auto_translate action failed: {e}")
+        return str(e), False
+
+
 _TYPE_COLORS = {
     "work":     "#5b8abf",  # blue
     "personal": "#a07ae0",  # purple
@@ -693,6 +980,7 @@ async def action_classify_events(owner: str, **kwargs) -> Tuple[str, bool]:
                     f"EVENTS: {_json.dumps(items)}"
                 )
                 try:
+                    await wait_for_interactive_quiet("calendar classification action")
                     raw = await llm_call_async_with_fallback(
                         llm_candidates,
                         messages=[{"role": "user", "content": prompt}],
@@ -761,19 +1049,44 @@ async def action_extract_email_events(owner: str, **kwargs) -> Tuple[str, bool]:
     import asyncio as _aio
     try:
         from routes.email_pollers import _run_auto_summarize_once
-        try:
-            # Hard wall-clock budget: 5 min total. Per-LLM call already has its own timeout.
-            result = await _aio.wait_for(
-                _run_auto_summarize_once(
-                    do_summary=False, do_reply=False, do_calendar=True, days_back=3,
-                ),
-                timeout=300,
+        account_id = _email_task_account_id(kwargs)
+        attempts = [
+            ("3d window, 3 emails", 3, 3, 240),
+            ("3d window, 2 emails", 3, 2, 150),
+            ("1d window, 1 email", 1, 1, 90),
+        ]
+        timed_out = []
+        last_result = ""
+        for label, days_back, max_process, timeout in attempts:
+            try:
+                result = await _aio.wait_for(
+                    _run_auto_summarize_once(
+                        do_summary=False,
+                        do_reply=False,
+                        do_calendar=True,
+                        days_back=days_back,
+                        account_id=account_id,
+                        max_process=max_process,
+                    ),
+                    timeout=timeout,
+                )
+                last_result = result or ""
+                if _result_is_config_error(result):
+                    return f"{result} ({label})", False
+                if _result_has_work(result):
+                    suffix = f"{label}" if not timed_out else f"{label}; retried after timeout"
+                    return f"{result} ({suffix})", True
+                raise TaskNoop(f"email→calendar: {result or 'no new emails'} ({label})")
+            except _aio.TimeoutError:
+                timed_out.append(label)
+                logger.warning(f"email calendar extraction timed out for {label}; retrying smaller batch")
+                continue
+        if timed_out:
+            raise TaskNoop(
+                "email→calendar: calendar extraction timed out on smaller batches; "
+                "will retry on the next scheduled run"
             )
-            if not _result_has_work(result):
-                raise TaskNoop(f"email→calendar: {result or 'no new emails'}")
-            return f"{result} (3d window)", True
-        except _aio.TimeoutError:
-            return "Email→calendar pass exceeded 5 min budget — try fewer emails or a faster model", False
+        raise TaskNoop(f"email→calendar: {last_result or 'no new emails'}")
     except Exception as e:
         logger.error(f"extract_email_events action failed: {e}")
         return str(e), False
@@ -942,6 +1255,7 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo
             )
 
             try:
+                await wait_for_interactive_quiet("sender signature action")
                 raw = await llm_call_async_with_fallback(
                     candidates,
                     messages=[{"role": "user", "content": prompt}],
@@ -1499,13 +1813,15 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
         STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
         AGE_CUTOFF = _dt.utcnow() - _td(days=7)
-        TRIAGE_VERSION = 3
+        TRIAGE_VERSION = 10
         CATEGORY_TAGS = {
-            "newsletter", "marketing", "notification", "finance", "bills",
-            "receipt", "travel", "security", "shopping", "social", "work",
-            "personal", "calendar",
+            "bills", "receipt", "travel", "calendar", "action-needed",
+        }
+        VISIBLE_EMAIL_TAGS = CATEGORY_TAGS | {"urgent", "reply-soon"}
+        MANAGED_TAGS = VISIBLE_EMAIL_TAGS | {
+            "newsletter", "marketing", "notification", "finance", "security",
+            "shopping", "social", "work", "personal", "legal", "support", "promo",
         }
-        MANAGED_TAGS = CATEGORY_TAGS | {"urgent", "reply-soon", "promo"}
 
         # ── 1. Resolve LLM candidates (utility primary + utility fallbacks; fall
         # through to default chat as a last resort).
@@ -1514,6 +1830,8 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
         if not candidates:
             return "No LLM endpoint available", False
 
+        target_account_id = _email_task_account_id(kwargs)
+
         # ── 2. Enumerate enabled accounts. Match this task's owner AND fall
         # back to the legacy "unowned account whose imap_user / from_address
         # == this owner" pattern — same rule `_get_email_config` uses, so a
@@ -1526,6 +1844,8 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                 unowned = _or(_EA.owner == None, _EA.owner == "")  # noqa: E711
                 same_mailbox = _or(_EA.imap_user == owner, _EA.from_address == owner)
                 q = q.filter(_or(_EA.owner == owner, _and(unowned, same_mailbox)))
+            if target_account_id:
+                q = q.filter(_EA.id == target_account_id)
             accounts = q.all()
         finally:
             db.close()
@@ -1534,12 +1854,95 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
 
         urgency_prompt = settings.get("urgent_email_prompt", "")
         per_uid_scores = {}   # key = "<acc_id>:<uid>" → {"score": 0-3, "reason": "..."}
-        all_unread_keys = set()  # for cache pruning
+        all_unread_keys = set()
         llm_attempts = 0
         saved_classifications = 0
         failed_classifications = []
+        tag_write_details = []
         scanned = 0
 
+        def _heuristic_email_verdict(item: dict) -> dict:
+            blob = (
+                f"{item.get('headers','')}\n{item.get('from','')}\n"
+                f"{item.get('subject','')}\n{item.get('body','')}"
+            ).lower()
+            response_tags = []
+            type_candidates = []
+
+            def add_response(tag: str):
+                if tag in CATEGORY_TAGS and tag not in response_tags:
+                    response_tags.append(tag)
+
+            def add_type(tag: str):
+                if tag in CATEGORY_TAGS and tag not in type_candidates:
+                    type_candidates.append(tag)
+
+            bulkish = bool(_re.search(
+                r"\b(list-unsubscribe|list-id|mailchimp|mailchimpapp|view this email in your browser|unsubscribe|newsletter|digest|precedence:\s*bulk)\b",
+                blob,
+            ))
+            marketingish = bool(_re.search(
+                r"\b(advertisement|sponsored|promo|promotion|sale|discount|offer|limited time|deal|coupon|shop now|buy now|membership|rewards?)\b",
+                blob,
+            ))
+            if bulkish or marketingish:
+                add_type("newsletter")
+            if _re.search(r"\b(receipt|order|注文|payment confirmation|delivery|shipment|tracking|お届け|購入)\b", blob):
+                add_type("receipt")
+            if _re.search(r"\b(bill|billing|amount due|overdue|pay by|payment due|subscription could not be renewed)\b", blob):
+                add_type("bills")
+            if _re.search(r"\b(court|charge|legal|lawyer|solicitor|claim|judgment|registration fee|debt)\b", blob):
+                add_type("legal")
+            if _re.search(r"\b(flight|hotel|booking|reservation|itinerary|train|ticket|trip|旅|予約)\b", blob):
+                add_type("travel")
+            if _re.search(r"\b(ticket|case|support|helpdesk|request)\b", blob):
+                add_type("support")
+            if _re.search(r"\b(meeting|appointment|calendar|invite|event|schedule|予定|保育園|連絡帳)\b", blob):
+                add_response("calendar")
+            if _re.search(
+                r"\b(action required|required action|please reply|please respond|deadline|by \d{1,2} |pay within|submit|sign|confirm|approval|waiting outside|locked out|can't get in|cannot get in|invoice|bill|billing|payment|balance|debt|subscription|renewal|overdue|amount due|court|charge|legal|lawyer|solicitor|claim|judgment)\b",
+                blob,
+            ):
+                add_response("action-needed")
+
+            type_priority = ("bills", "receipt", "travel")
+            tags = [*response_tags]
+            for type_tag in type_priority:
+                if type_tag in type_candidates and type_tag not in tags:
+                    tags.append(type_tag)
+                if len(tags) >= len(response_tags) + 2:
+                    break
+
+            score = 0
+            reason = "categorized by email metadata"
+            if "action-needed" in response_tags:
+                score = 2
+                reason = "action likely needed"
+            if _re.search(r"\b(urgent|immediately|final notice|locked out|waiting outside|can't get in|cannot get in)\b", blob):
+                score = 3
+                reason = "urgent wording"
+            if (bulkish or marketingish) and score < 2:
+                score = 0
+                reason = "bulk marketing/newsletter"
+
+            _from_raw = item.get("from", "") or ""
+            if "<" in _from_raw:
+                _from_short = _from_raw.split("<", 1)[0].strip().strip('"') or _from_raw
+            else:
+                _from_short = _from_raw
+            return {
+                "score": max(0, min(3, score)),
+                "tags": tags[:4],
+                "spam": False,
+                "reason": reason,
+                "subject": (item.get("subject") or "")[:200],
+                "from": _from_short[:120],
+                "triage_version": TRIAGE_VERSION,
+                "message_id": (item.get("message_id") or "").strip(),
+                "unread": bool(item.get("unread")),
+                "ts": _time.time(),
+            }
+
         # ── 3. Per-account scan: pull headers + lightweight body for new UIDs
         # since 7 days ago, score via LLM, cache the verdict.
         for acc in accounts:
@@ -1555,13 +1958,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                 conn = _imap_connect(account.id)
                 try:
                     conn.select("INBOX", readonly=True)
-                    # IMAP date is the only practical pre-filter — UNSEEN AND
-                    # SINCE 7-days-ago. Date format is DD-Mon-YYYY.
+                    # Tag recent inbox mail, not only unread mail. Urgency
+                    # reminders below still only notify for unread messages.
                     since_str = AGE_CUTOFF.strftime("%d-%b-%Y")
-                    status, data = conn.search(None, f'(UNSEEN SINCE {since_str})')
+                    status, data = conn.uid("SEARCH", None, f'(SINCE {since_str})')
                     if status != "OK" or not data or not data[0]:
                         return results
-                    uids = data[0].split()
+                    uids = data[0].split()[-30:]
                     for uid_b in uids:
                         uid = uid_b.decode() if isinstance(uid_b, bytes) else str(uid_b)
                         key = f"{account.id}:{uid}"
@@ -1573,9 +1976,14 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                             continue
                         # Pull headers + first ~800 chars of plaintext body.
                         try:
-                            st, msg_data = conn.fetch(uid_b, "(RFC822.HEADER BODY.PEEK[TEXT]<0.800>)")
+                            st, msg_data = conn.uid("FETCH", uid_b, "(UID FLAGS RFC822.HEADER BODY.PEEK[TEXT]<0.800>)")
                             if st != "OK" or not msg_data:
                                 continue
+                            flags_blob = b" ".join(
+                                part[0] for part in msg_data
+                                if isinstance(part, tuple) and part and isinstance(part[0], (bytes, bytearray))
+                            )
+                            is_unread = b"\\Seen" not in flags_blob
                             # Headers + body land in different tuples in the
                             # response — concatenate the bytes for parsing.
                             raw = b""
@@ -1635,6 +2043,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                                 "headers": header_blob,
                                 "body": body_snippet.strip(),
                                 "message_id": (msg.get("Message-ID") or "").strip(),
+                                "unread": is_unread,
                             })
                         except Exception as _fe:
                             logger.debug(f"urgency: header fetch for uid {uid} failed: {_fe}")
@@ -1652,25 +2061,33 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
             for item in items:
                 scanned += 1
                 key = item["key"]
-                all_unread_keys.add(key)
+                if item.get("unread"):
+                    all_unread_keys.add(key)
                 if item.get("cached"):
-                    per_uid_scores[key] = item["cached"]
+                    cached_v = dict(item["cached"])
+                    cached_v["unread"] = bool(item.get("unread"))
+                    per_uid_scores[key] = cached_v
                     continue
                 # Skip uids we couldn't fetch (no subject/from/body).
                 if not item.get("subject") and not item.get("from"):
                     continue
+                verdict = _heuristic_email_verdict(item)
+                cache.setdefault("uids", {})[item["uid"]] = verdict
+                per_uid_scores[key] = verdict
+                saved_classifications += 1
+                continue
                 # ── LLM-classify. JSON-only response; bullet-proof parse.
                 llm_attempts += 1
                 prompt = (
-                    "You are triaging ONE unread email. Return ONLY JSON: "
+                    "You are triaging ONE email. Return ONLY JSON: "
                     "{\"score\":0|1|2|3,\"tags\":[\"...\"],\"spam\":false,"
                     "\"reason\":\"one short phrase\"}.\n"
                     "0 = trivial / promotional · 1 = informational, no reply needed · "
                     "2 = should reply within a day · 3 = urgent, reply now (deadline, blocker).\n\n"
-                    "Allowed tags: newsletter, marketing, notification, finance, bills, receipt, "
-                    "travel, security, shopping, social, work, personal, calendar.\n"
-                    "Use marketing for ads, promos, sales, offers, and cold sales. Use newsletter "
-                    "for newsletters, digests, and recurring content. spam=true for scams, phishing, "
+                    "Allowed visible tags: urgent, reply-soon, action-needed, calendar, bills, receipt, travel.\n"
+                    "Use action-needed when the user likely needs to reply, pay, sign, book, or decide. "
+                    "Use bills for bills or debts, receipt for purchases/deliveries, travel for reservations/trips, "
+                    "and calendar only when a calendar event/reminder is involved. spam=true for scams, phishing, "
                     "junk, cold sales, generic ads, or no-personal-action bulk mail.\n"
                     "Important: 'I'm outside', 'I am outside', 'waiting outside', 'at the door', "
                     "'locked out', or 'can't get in' means score 3 unless clearly historical.\n\n"
@@ -1679,6 +2096,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                     f"Snippet:\n{item.get('body','')}\n"
                 )
                 try:
+                    await wait_for_interactive_quiet("email urgency action")
                     raw = await llm_call_async_with_fallback(
                         candidates,
                         [{"role": "user", "content": prompt}],
@@ -1739,14 +2157,10 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         r"\b(advertisement|sponsored|promo|promotion|sale|discount|offer|limited time|deal|tickets?|tour|merch|stream|purchase|sold out|low tickets|coupon|shop now|buy now)\b",
                         _blob,
                     ))
-                    if "newsletter" not in tags and bulkish:
-                        tags.append("newsletter")
-                    if "marketing" not in tags and marketingish:
-                        tags.append("marketing")
                     if (bulkish or marketingish) and score < 2:
                         score = 0
                         if not reason or "urgent" in reason.lower():
-                            reason = "Bulk marketing/newsletter; no personal reply needed"
+                            reason = "bulk mail; no personal reply needed"
                     # Strip "Name <addr>" to bare display name for compact summary.
                     _from_raw = item.get("from", "") or ""
                     if "<" in _from_raw:
@@ -1764,6 +2178,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         # Cache the message_id too so re-scans of already-cached
                         # UIDs can still write the inbox tag without re-LLM'ing.
                         "message_id": (item.get("message_id") or "").strip(),
+                        "unread": bool(item.get("unread")),
                         "ts": _time.time(),
                     }
                     cache.setdefault("uids", {})[item["uid"]] = verdict
@@ -1778,9 +2193,9 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                     logger.debug(f"urgency: LLM classify failed for {key}: {e}")
                     continue
 
-            # ── Prune cache entries for UIDs that are no longer unread (replied
-            # / archived / deleted). Compare against `items` (everything UNSEEN
-            # in this scan window).
+            # ── Prune cache entries for UIDs that are no longer in the recent
+            # scan window. Read messages remain cached because tags are useful
+            # on read mail too; unread state is refreshed per scan above.
             seen_uids = {it["uid"] for it in items}
             cache_uids = cache.get("uids", {})
             for stale in [u for u in cache_uids if u not in seen_uids]:
@@ -1815,15 +2230,17 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         _tag = str(_tag).strip().lower().replace("_", "-")
                         if _tag == "promo":
                             _tag = "marketing"
-                        if _tag in CATEGORY_TAGS and _tag not in _new_tags:
+                        if _tag == "action-needed" and any(t in _new_tags for t in ("urgent", "reply-soon")):
+                            continue
+                        if _tag in VISIBLE_EMAIL_TAGS and _tag not in _new_tags:
                             _new_tags.append(_tag)
                     _spam = 1 if _v.get("spam") else 0
                     # _key is "<account_id>:<uid>" — extract uid for the row.
-                    _uid_only = _key.split(":", 1)[-1]
+                    _acc_id, _uid_only = (_key.split(":", 1) + [""])[:2]
                     _owner_key = owner or ""
                     _row = _conn.execute(
-                        "SELECT tags FROM email_tags WHERE message_id=? AND owner=?",
-                        (_msg_id, _owner_key),
+                        "SELECT tags FROM email_tags WHERE message_id=? AND owner=? AND account_id=?",
+                        (_msg_id, _owner_key, _acc_id),
                     ).fetchone()
                     if _row:
                         try:
@@ -1842,23 +2259,42 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         for _tag in _new_tags:
                             if _tag not in _existing:
                                 _existing.append(_tag)
+                        if _new_tags or _spam:
+                            tag_write_details.append({
+                                "uid": _uid_only,
+                                "subject": _v.get("subject", ""),
+                                "from": _v.get("from", ""),
+                                "tags": list(_new_tags),
+                                "spam": _spam,
+                                "reason": _v.get("reason", ""),
+                                "updated": True,
+                            })
                         _conn.execute(
                             "UPDATE email_tags SET tags=?, spam_verdict=?, spam_reason=?, uid=?, folder=?, subject=?, sender=? "
-                            "WHERE message_id=? AND owner=?",
+                            "WHERE message_id=? AND owner=? AND account_id=?",
                             (_json.dumps(_existing), _spam, _v.get("reason", ""), _uid_only, "INBOX",
-                             _v.get("subject", ""), _v.get("from", ""), _msg_id, _owner_key),
+                             _v.get("subject", ""), _v.get("from", ""), _msg_id, _owner_key, _acc_id),
                         )
                     else:
                         if not _new_tags and not _spam:
                             continue
                         _conn.execute(
                             "INSERT INTO email_tags "
-                            "(message_id, owner, uid, folder, subject, sender, tags, spam_verdict, spam_reason, created_at) "
-                            "VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?)",
-                            (_msg_id, _owner_key, _uid_only, _v.get("subject", ""),
+                            "(message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict, spam_reason, created_at) "
+                            "VALUES (?, ?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?)",
+                            (_msg_id, _owner_key, _acc_id, _uid_only, _v.get("subject", ""),
                              _v.get("from", ""), _json.dumps(_new_tags), _spam, _v.get("reason", ""),
                              _dt2.utcnow().isoformat()),
                         )
+                        tag_write_details.append({
+                            "uid": _uid_only,
+                            "subject": _v.get("subject", ""),
+                            "from": _v.get("from", ""),
+                            "tags": list(_new_tags),
+                            "spam": _spam,
+                            "reason": _v.get("reason", ""),
+                            "updated": False,
+                        })
                 _conn.commit()
             finally:
                 _conn.close()
@@ -1866,7 +2302,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
             logger.warning(f"urgency: bulk tag write failed: {_te}")
 
         # ── 4. Aggregate state. urgent = score ≥ 2.
-        urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2]
+        urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2 and v.get("unread")]
         max_score = max((v.get("score", 0) for v in per_uid_scores.values()), default=0)
         total_urgent = len(urgent_keys)
 
@@ -1975,13 +2411,28 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
             f"reply-soon {tier_counts[2]} · info {tier_counts[1]} · trivial {tier_counts[0]} · "
             f"{saved_classifications} saved classifications"
         )
-        if llm_attempts != saved_classifications:
-            head += f" · {llm_attempts - saved_classifications} failed"
+        if failed_classifications:
+            head += f" · {len(failed_classifications)} failed"
         if newly_notified:
             head += f" · notified {len(newly_notified)}"
         if notify_failed:
             head += f" · notify failed {len(notify_failed)}"
 
+        def _fmt_tag_write(v):
+            subj = (v.get("subject") or "(no subject)")[:80]
+            frm = v.get("from") or ""
+            tags = list(v.get("tags") or [])
+            if v.get("spam"):
+                tags.append("spam")
+            tag_txt = ", ".join(tags) if tags else "cleared managed tags"
+            why = v.get("reason") or ""
+            op = "updated" if v.get("updated") else "created"
+            line = f"- **{subj}**" + (f" — _{frm}_" if frm else "")
+            line += f" — `{tag_txt}` ({op})"
+            if why:
+                line += f" · {why}"
+            return line
+
         def _fmt_one(v, newly_notified_set, failed_set, key):
             subj = (v.get("subject") or "(no subject)")[:80]
             frm = v.get("from") or ""
@@ -1997,6 +2448,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
         for k, v in per_uid_scores.items():
             by_tier.setdefault(v.get("score", 0), []).append((k, v))
         lines = [head]
+        if tag_write_details:
+            lines.append("")
+            lines.append(f"**Applied tags ({len(tag_write_details)}):**")
+            for v in tag_write_details[:16]:
+                lines.append(_fmt_tag_write(v))
+            if len(tag_write_details) > 16:
+                lines.append(f"…and {len(tag_write_details) - 16} more")
         tier_labels = {3: "Urgent", 2: "Reply soon", 1: "Informational", 0: "Trivial"}
         for tier in (3, 2, 1, 0):
             items_t = by_tier.get(tier, [])
@@ -2068,6 +2526,7 @@ async def action_cookbook_serve(
         end_after_min = int(cfg.get("end_after_min") or 0)
     except Exception:
         end_after_min = 0
+    set_default = bool(cfg.get("set_default", True))
 
     state_path = Path(COOKBOOK_STATE_FILE)
     try:
@@ -2154,6 +2613,51 @@ async def action_cookbook_serve(
         return f"Launch rejected: {data.get('error') or data.get('detail') or 'unknown'}", False
 
     sid = data.get("session_id") or ""
+    endpoint_id = data.get("endpoint_id") or ""
+    # Scheduled serves are usually meant to become the active local model for
+    # chat/tools while their time window is open. Persist both endpoint and
+    # model so task/utility/default resolution does not keep routing to a stale
+    # API fallback. Allow explicit opt-out with {"set_default": false}.
+    if endpoint_id and set_default:
+        try:
+            selected_model = repo_id
+            try:
+                from core.database import SessionLocal as _SL, ModelEndpoint as _ME
+                _db = _SL()
+                try:
+                    _ep = _db.query(_ME).filter(_ME.id == endpoint_id).first()
+                    if _ep and _ep.cached_models:
+                        _models = json.loads(_ep.cached_models or "[]")
+                        if isinstance(_models, list) and _models:
+                            selected_model = str(_models[0])
+                finally:
+                    _db.close()
+            except Exception:
+                pass
+            from src.settings import load_settings as _load_settings, save_settings as _save_settings
+            _settings = _load_settings()
+            _settings["default_endpoint_id"] = endpoint_id
+            _settings["default_model"] = selected_model
+            # Keep background tasks aligned unless the user explicitly chose a
+            # separate task model.
+            if not (_settings.get("task_endpoint_id") or "").strip():
+                _settings["task_endpoint_id"] = endpoint_id
+                _settings["task_model"] = selected_model
+            if not (_settings.get("utility_endpoint_id") or "").strip():
+                _settings["utility_endpoint_id"] = endpoint_id
+                _settings["utility_model"] = selected_model
+            _save_settings(_settings)
+            if owner:
+                from routes.prefs_routes import _load_for_user, _save_for_user
+                _prefs = _load_for_user(owner)
+                _prefs["default_endpoint_id"] = endpoint_id
+                _prefs["default_model"] = selected_model
+                if not (_prefs.get("utility_endpoint_id") or "").strip():
+                    _prefs["utility_endpoint_id"] = endpoint_id
+                    _prefs["utility_model"] = selected_model
+                _save_for_user(owner, _prefs)
+        except Exception as e:
+            logger.warning(f"cookbook_serve: default endpoint update failed: {e}")
     # Register the new task in cookbook_state.json + stamp it with our
     # scheduler-owner markers. /api/model/serve spawns the tmux session
     # but leaves the state-write to the UI — when a scheduled action
@@ -2197,12 +2701,16 @@ async def action_cookbook_serve(
                     "sshPort": ssh_port or "",
                     "platform": platform or "linux",
                     "_serveReady": False,
-                    "_endpointAdded": False,
+                    "_endpointAdded": bool(endpoint_id),
                 }
                 tasks.append(existing)
             # Stamp ownership + end-at on the task entry.
             existing["_scheduledByTask"] = task_name or ""
             existing["_scheduledByOwner"] = owner or ""
+            if endpoint_id:
+                existing["_endpointId"] = endpoint_id
+                existing["endpointId"] = endpoint_id
+                existing["_endpointAdded"] = True
             if end_after_min > 0:
                 existing["_scheduledStopAtMs"] = int(_time.time() * 1000) + end_after_min * 60 * 1000
             fresh["tasks"] = tasks
@@ -2230,6 +2738,7 @@ BUILTIN_ACTIONS = {
     "tidy_research": action_tidy_research,
     "summarize_emails": action_summarize_emails,
     "draft_email_replies": action_draft_email_replies,
+    "email_auto_translate": action_email_auto_translate,
     "extract_email_events": action_extract_email_events,
     "classify_events": action_classify_events,
     # ping_events removed from the user-facing registry. Calendar reminders
@@ -2254,6 +2763,7 @@ BUILTIN_ACTION_INFO = {
     "tidy_research": "Remove orphaned research files (sessions that were deleted)",
     "summarize_emails": "Pre-generate AI summaries for new inbox emails",
     "draft_email_replies": "Pre-draft AI reply suggestions for new inbox emails",
+    "email_auto_translate": "Detect foreign-language emails and cache translated text for the email reader",
     "extract_email_events": "Scan emails for booking/meeting confirmations and auto-add to calendar",
     "classify_events": "Tag upcoming events with importance (low/normal/high/critical) and type (work/health/travel/etc.); colors them too",
     "daily_brief": "Build a morning digest: today's calendar, unread email count + top senders, active todos",

src/caldav_sync.py

@@ -25,6 +25,7 @@ Design notes:
 import asyncio
 import hashlib
 import ipaddress
+import json
 import logging
 import os
 import socket
@@ -500,6 +501,7 @@ def _event_payload(ev) -> dict:
         "all_day": ev.all_day,
         "is_utc": ev.is_utc,
         "rrule": ev.rrule or "",
+        "recurrence_exdates": json.loads(ev.recurrence_exdates or "[]") if getattr(ev, "recurrence_exdates", "") else [],
     }
 
 

src/caldav_writeback.py

@@ -33,7 +33,8 @@ def build_event_ical(ev: dict) -> str:
     """Serialize a local event dict to a VCALENDAR/VEVENT iCalendar string.
 
     ``ev`` keys: uid, summary, description, location, dtstart (datetime),
-    dtend (datetime), all_day (bool), is_utc (bool), rrule (str).
+    dtend (datetime), all_day (bool), is_utc (bool), rrule (str),
+    recurrence_exdates (list[str]).
     Mirrors how the pull path interprets is_utc/all_day so a round-trip is stable.
     """
     from icalendar import Calendar, Event as iEvent
@@ -70,6 +71,15 @@ def build_event_ical(ev: dict) -> str:
             ve.add("rrule", vRecur.from_ical(ev["rrule"]))
         except Exception:
             logger.debug("CalDAV write-back: skipping unparseable rrule %r", ev.get("rrule"))
+    for exdate in ev.get("recurrence_exdates") or []:
+        try:
+            if ev.get("all_day"):
+                ve.add("exdate", datetime.strptime(exdate[:10], "%Y-%m-%d").date())
+            else:
+                dt = datetime.strptime(exdate[:16], "%Y-%m-%dT%H:%M")
+                ve.add("exdate", dt.replace(tzinfo=timezone.utc) if ev.get("is_utc") else dt)
+        except Exception:
+            logger.debug("CalDAV write-back: skipping unparseable exdate %r", exdate)
 
     cal.add_component(ve)
     return cal.to_ical().decode("utf-8")

src/chat_handler.py

@@ -29,6 +29,34 @@ from src.youtube_handler import (
 logger = logging.getLogger(__name__)
 
 
+def _sync_upload_vision_to_gallery(file_info: Dict[str, Any], owner: Optional[str], text: str) -> None:
+    file_hash = (file_info or {}).get("hash")
+    if not file_hash or not text:
+        return
+    try:
+        from core.database import GalleryImage, SessionLocal
+        db = SessionLocal()
+        try:
+            q = db.query(GalleryImage).filter(
+                GalleryImage.file_hash == file_hash,
+                GalleryImage.is_active == True,  # noqa: E712
+            )
+            if owner:
+                q = q.filter(GalleryImage.owner == owner)
+            img = q.first()
+            if not img:
+                return
+            img.caption = text.strip()
+            db.commit()
+        except Exception:
+            db.rollback()
+            raise
+        finally:
+            db.close()
+    except Exception as e:
+        logger.warning("Failed to sync upload vision text to gallery: %s", e)
+
+
 class ChatHandler:
     """Handles chat operations for both streaming and non-streaming endpoints."""
 
@@ -207,6 +235,7 @@ class ChatHandler:
                                     _vtext = _vf.read().strip()
                                 if _vtext:
                                     enhanced_message += f"\n[User-corrected caption / OCR for this image — treat as authoritative]:\n{_vtext}"
+                                    _sync_upload_vision_to_gallery(file_info, owner, _vtext)
                                     _m = meta_by_id.get(att_id)
                                     if _m is not None:
                                         _m["vision"] = _vtext
@@ -226,6 +255,7 @@ class ChatHandler:
                                     cached_desc = _vf.read().strip()
                                 if cached_desc and not cached_desc.startswith("["):
                                     vl_desc = cached_desc
+                                    _sync_upload_vision_to_gallery(file_info, owner, vl_desc)
                             except Exception:
                                 vl_desc = None
                         if not vl_desc:
@@ -237,6 +267,7 @@ class ChatHandler:
                                     os.makedirs(os.path.join(UPLOAD_DIR, ".vision"), exist_ok=True)
                                     with open(_vcache, "w", encoding="utf-8") as _vf:
                                         _vf.write(vl_desc)
+                                    _sync_upload_vision_to_gallery(file_info, owner, vl_desc)
                                 except Exception:
                                     pass
                         enhanced_message = f"{enhanced_message}\n\n[Image: {file_info['name']}]\n{vl_desc}"

src/context_budget.py

@@ -18,6 +18,13 @@ DEFAULT_BUDGET = 6000
 DEFAULT_HEADROOM = 0.85
 
 
+def _int_or_zero(value) -> int:
+    try:
+        return int(value or 0)
+    except (TypeError, ValueError):
+        return 0
+
+
 def compute_input_token_budget(
     configured: int,
     context_length: int,
@@ -48,8 +55,8 @@ def compute_input_token_budget(
         - When the window is unknown (context_length <= 0), use the conservative
           ``default`` budget and do NOT scale off the fallback.
     """
-    configured = int(configured or 0)
-    context_length = int(context_length or 0)
+    configured = _int_or_zero(configured)
+    context_length = _int_or_zero(context_length)
 
     if explicit and configured > 0:
         return min(configured, context_length) if context_length > 0 else configured

src/cookbook_serve_lifecycle.py

@@ -37,6 +37,13 @@ async def _delete_endpoint_for_task(task: dict) -> None:
     the picker (probe goes offline; chats still try to route there) and
     the user has to delete it by hand in Settings -> Endpoints.
     """
+    endpoint_id = (task.get("_endpointId") or task.get("endpointId") or "").strip()
+    if not endpoint_id:
+        logger.info(
+            "cookbook_serve_lifecycle: task %s has no endpoint id; skipping endpoint deletion",
+            task.get("sessionId") or task.get("id") or "",
+        )
+        return
     import re as _re
     payload = task.get("payload") or {}
     cmd = str(payload.get("_cmd") or "")
@@ -66,13 +73,10 @@ async def _delete_endpoint_for_task(task: dict) -> None:
             if r.status_code >= 400:
                 return
             eps = r.json() if r.content else []
-            # Prefer exact URL match; fall back to host:port substring so we
-            # still catch the case where 0.0.0.0 vs the registered host
-            # representation diverged.
-            ep = next((e for e in eps if e.get("base_url") == base_url), None)
-            if not ep:
-                hostport = f"{host}:{port}"
-                ep = next((e for e in eps if hostport in (e.get("base_url") or "")), None)
+            # Delete only the endpoint created by this scheduled serve. URL
+            # matching is unsafe because a later scheduled serve can reuse the
+            # same host:port after an older task has gone stale.
+            ep = next((e for e in eps if e.get("id") == endpoint_id), None)
             if ep:
                 await client.delete(
                     f"{internal_api_base()}/api/model-endpoints/{ep['id']}",

src/document_actions.py

@@ -6,7 +6,7 @@ Reusable document actions callable from both REST routes and the task scheduler.
 
 import logging
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 
 logger = logging.getLogger(__name__)
 
@@ -77,10 +77,28 @@ async def run_document_tidy(owner: str) -> str:
         deleted = 0
         kept = 0
         survivors = []  # docs that pass the junk rules, considered for dedup
+        now = datetime.now(timezone.utc)
 
         for doc in docs:
+            created = doc.created_at
+            if created and created.tzinfo is None:
+                created = created.replace(tzinfo=timezone.utc)
+
+            # Skip freshly created documents to avoid deleting them while the user is actively editing
+            if created and (now - created).total_seconds() < 900:  # 15 minutes
+                survivors.append(doc)
+                continue
+
             content = (doc.current_content or "").strip()
             title = (doc.title or "").strip().lower()
+            is_fresh_empty = (
+                not content
+                and created is not None
+                and (now - created).total_seconds() < 1800
+            )
+            if is_fresh_empty:
+                survivors.append(doc)
+                continue
 
             # Strip markdown noise to get "real" character count
             stripped = re.sub(r"^#{1,6}\s+", "", content, flags=re.MULTILINE)  # headers

src/host_docker_access.py

@@ -0,0 +1,62 @@
+"""Policy checks for explicit host Docker access from a container."""
+
+import os
+import stat
+from collections.abc import Mapping
+
+
+HOST_DOCKER_ENV_VAR = "ODYSSEUS_ENABLE_HOST_DOCKER"
+HOST_DOCKER_SOCKET_PATH = "/var/run/docker.sock"
+
+HOST_DOCKER_ACCESS_HINT = (
+    "Local Docker daemon access is disabled inside the Odysseus container; a "
+    "Docker CLI alone is not enough. Default Docker Compose intentionally does "
+    "not mount the host Docker socket. Raw socket access is high-trust and can "
+    "grant broad control over the host Docker daemon. If you accept that risk, "
+    "enable docker/host-docker.yml. Remote server Docker workflows over SSH "
+    "remain preferred."
+)
+
+
+def running_in_container(
+    dockerenv_path: str = "/.dockerenv",
+    cgroup_path: str = "/proc/1/cgroup",
+) -> bool:
+    if os.path.exists(dockerenv_path):
+        return True
+    try:
+        with open(cgroup_path, "r", encoding="utf-8") as handle:
+            contents = handle.read()
+    except OSError:
+        return False
+    return any(token in contents for token in ("docker", "containerd", "kubepods"))
+
+
+def host_docker_access_enabled(
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+    *,
+    environ: Mapping[str, str] | None = None,
+) -> bool:
+    env = os.environ if environ is None else environ
+    if env.get(HOST_DOCKER_ENV_VAR, "").strip().lower() != "true":
+        return False
+    try:
+        mode = os.stat(socket_path).st_mode
+    except OSError:
+        return False
+    return stat.S_ISSOCK(mode)
+
+
+def local_docker_available(
+    *,
+    cli_available: bool,
+    in_container: bool | None = None,
+    environ: Mapping[str, str] | None = None,
+    socket_path: str = HOST_DOCKER_SOCKET_PATH,
+) -> bool:
+    if not cli_available:
+        return False
+    containerized = running_in_container() if in_container is None else in_container
+    if not containerized:
+        return True
+    return host_docker_access_enabled(socket_path, environ=environ)

src/interactive_gate.py

@@ -0,0 +1,193 @@
+"""Foreground activity gate for background work.
+
+Background tasks are allowed to run only after normal UI/API traffic has
+settled. This keeps scheduled jobs and email pollers from competing with the
+user opening Odysseus, Cookbook, email, documents, notes, or other panels.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager
+import os
+import time
+
+
+_ACTIVE_REQUESTS = 0
+_LAST_ACTIVITY = 0.0
+_LAST_BROWSER_ACTIVITY = 0.0
+_COND: asyncio.Condition | None = None
+
+
+def _enabled() -> bool:
+    return os.getenv("BACKGROUND_TASK_FOREGROUND_GATE", "true").lower() not in {"0", "false", "no", "off"}
+
+
+def _quiet_seconds() -> float:
+    try:
+        return max(0.0, float(os.getenv("BACKGROUND_TASK_QUIET_MS", "1500")) / 1000.0)
+    except Exception:
+        return 1.5
+
+
+def _max_wait_seconds() -> float:
+    """0 means wait indefinitely until the UI is quiet."""
+    try:
+        return max(0.0, float(os.getenv("BACKGROUND_TASK_MAX_WAIT_SECONDS", "0")))
+    except Exception:
+        return 0.0
+
+
+def _browser_active_seconds() -> float:
+    """How long a visible Odysseus browser heartbeat blocks background tasks."""
+    try:
+        return max(0.0, float(os.getenv("BACKGROUND_TASK_BROWSER_ACTIVE_SECONDS", "45")))
+    except Exception:
+        return 45.0
+
+
+def _condition() -> asyncio.Condition:
+    global _COND
+    if _COND is None:
+        _COND = asyncio.Condition()
+    return _COND
+
+
+_PASSIVE_EXACT_PATHS = {
+    "/api/activity/heartbeat",
+    "/api/tasks/notifications",
+    "/api/research/active",
+    "/api/email/urgency-state",
+}
+
+_PASSIVE_PREFIXES = (
+    "/api/chat/stream_status",
+    "/api/health",
+    "/api/prefs",
+)
+
+
+def should_track_interactive_request(path: str, method: str = "GET") -> bool:
+    if not _enabled():
+        return False
+    if (method or "").upper() == "OPTIONS":
+        return False
+    if path in _PASSIVE_EXACT_PATHS:
+        return False
+    if any(path.startswith(prefix) for prefix in _PASSIVE_PREFIXES):
+        return False
+    return True
+
+
+async def mark_browser_activity() -> None:
+    """Record that an authenticated browser tab is visibly using Odysseus."""
+    global _LAST_BROWSER_ACTIVITY
+    if not _enabled():
+        return
+    cond = _condition()
+    async with cond:
+        _LAST_BROWSER_ACTIVITY = time.monotonic()
+        cond.notify_all()
+
+
+def _has_recent_browser_activity(now: float | None = None) -> bool:
+    ttl = _browser_active_seconds()
+    if ttl <= 0 or _LAST_BROWSER_ACTIVITY <= 0:
+        return False
+    return ((now if now is not None else time.monotonic()) - _LAST_BROWSER_ACTIVITY) < ttl
+
+
+def has_foreground_activity(now: float | None = None) -> bool:
+    """Return True when foreground browser/model work should stop background jobs.
+
+    This is intentionally narrower than `wait_for_interactive_quiet`: active
+    request tracking is good for delaying task startup, but a running task
+    should not cancel itself just because the UI polls a passive endpoint.
+    Browser heartbeats and active chat streams are the durable "user is here"
+    signals.
+    """
+    if not _enabled():
+        return False
+    t = now if now is not None else time.monotonic()
+    return _has_recent_browser_activity(t) or _has_active_chat_stream()
+
+
+def _has_active_chat_stream() -> bool:
+    """Best-effort check for foreground model work that outlives HTTP requests.
+
+    Chat/agent streams are detached from the browser SSE so a stream can keep
+    running after the request that started it has returned. Background LLM
+    tasks must still wait for those runs; otherwise helpers like email
+    auto-translate compete with the user's active chat on the same local model.
+    """
+    try:
+        from routes import chat_routes as _chat_routes
+        active_streams = getattr(_chat_routes, "_active_streams", {}) or {}
+        if active_streams:
+            return True
+    except Exception:
+        pass
+    try:
+        from src import agent_runs
+        runs = getattr(agent_runs, "_RUNS", {}) or {}
+        return any(getattr(run, "status", None) == "running" for run in runs.values())
+    except Exception:
+        return False
+
+
+@asynccontextmanager
+async def track_interactive_request(path: str = "", method: str = ""):
+    global _ACTIVE_REQUESTS, _LAST_ACTIVITY
+    if not _enabled():
+        yield
+        return
+
+    cond = _condition()
+    async with cond:
+        _ACTIVE_REQUESTS += 1
+        _LAST_ACTIVITY = time.monotonic()
+        cond.notify_all()
+    try:
+        yield
+    finally:
+        async with cond:
+            _ACTIVE_REQUESTS = max(0, _ACTIVE_REQUESTS - 1)
+            _LAST_ACTIVITY = time.monotonic()
+            cond.notify_all()
+
+
+async def wait_for_interactive_quiet(label: str = "") -> bool:
+    """Wait until foreground requests have stopped for the configured window.
+
+    Returns True if the caller had to wait at all. The label is intentionally
+    only for future logging/debugging so callers can keep their code simple.
+    """
+    if not _enabled():
+        return False
+
+    quiet = _quiet_seconds()
+    max_wait = _max_wait_seconds()
+    deadline = time.monotonic() + max_wait if max_wait > 0 else None
+    cond = _condition()
+    waited = False
+
+    while True:
+        async with cond:
+            now = time.monotonic()
+            quiet_remaining = quiet - (now - _LAST_ACTIVITY)
+            active_stream = _has_active_chat_stream()
+            browser_active = _has_recent_browser_activity(now)
+            if _ACTIVE_REQUESTS <= 0 and quiet_remaining <= 0 and not active_stream and not browser_active:
+                return waited
+
+            waited = True
+            timeout = 0.25 if (_ACTIVE_REQUESTS > 0 or active_stream or browser_active) else min(max(quiet_remaining, 0.05), 0.5)
+            if deadline is not None:
+                remaining = deadline - now
+                if remaining <= 0:
+                    return waited
+                timeout = min(timeout, remaining)
+            try:
+                await asyncio.wait_for(cond.wait(), timeout=timeout)
+            except asyncio.TimeoutError:
+                pass

src/llm_core.py

@@ -110,6 +110,18 @@ _HARMONY_MARKERS = (
 )
 _HARMONY_MAX_MARKER_LEN = max(len(marker) for marker in _HARMONY_MARKERS)
 
+_VISIBLE_CHAT_TEMPLATE_ARTIFACT_RE = re.compile(
+    r"(?:\|end\|)+\|?assistan(?:t)?\|?"
+    r"|\|assistan(?:t)?\|"
+    r"|<\|im_start\|>\s*assistant"
+    r"|<\|im_end\|>",
+    re.IGNORECASE,
+)
+
+
+def _strip_visible_chat_template_artifacts(text: str) -> str:
+    return _VISIBLE_CHAT_TEMPLATE_ARTIFACT_RE.sub("", text or "")
+
 
 def _harmony_suffix_hold_len(text: str) -> int:
     """Return how many trailing chars could be the start of a harmony marker."""
@@ -345,6 +357,18 @@ def _normalize_ollama_url(url: str) -> str:
     return base.rstrip("/") + "/chat"
 
 
+def _normalize_openai_chat_url(url: str) -> str:
+    """Ensure an OpenAI-compatible base URL points at /chat/completions."""
+    base = (url or "").strip().rstrip("/")
+    if not base:
+        return base
+    if base.endswith("/chat/completions") or base.endswith("/completions"):
+        return base
+    if base.endswith("/models"):
+        base = base[: -len("/models")].rstrip("/")
+    return base + "/chat/completions"
+
+
 def _ollama_normalize_messages(messages: List[Dict]) -> List[Dict]:
     """Adapt Odysseus' canonical OpenAI-style messages to native Ollama /api/chat.
 
@@ -1361,6 +1385,7 @@ def _sanitize_llm_messages(messages: List[Dict]) -> List[Dict]:
 
     return merged
 
+
 def _normalize_anthropic_url(url: str) -> str:
     """Ensure Anthropic URL points to /v1/messages."""
     url = url.rstrip("/")
@@ -1563,7 +1588,7 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
             stream=False, num_ctx=get_context_length(url, model),
         )
     else:
-        target_url = url
+        target_url = _normalize_openai_chat_url(url)
         if provider == "copilot":
             from src.copilot import apply_request_headers
             apply_request_headers(h, messages_copy)
@@ -1767,7 +1792,7 @@ async def llm_call_async(
             stream=False, num_ctx=get_context_length(url, model),
         )
     else:
-        target_url = url
+        target_url = _normalize_openai_chat_url(url)
         h = _provider_headers(provider, headers)
         if provider == "copilot":
             from src.copilot import apply_request_headers
@@ -1845,7 +1870,8 @@ async def llm_call_async(
 async def stream_llm(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
                      max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
                      timeout: int = LLMConfig.STREAM_TIMEOUT, prompt_type: Optional[str] = None,
-                     tools: Optional[List[Dict]] = None, session_id: Optional[str] = None):
+                     tools: Optional[List[Dict]] = None, session_id: Optional[str] = None,
+                     tool_choice_none: bool = False):
     """Stream LLM responses with improved error handling.
 
     Yields SSE chunks:
@@ -1889,7 +1915,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
         h = _provider_headers(provider, headers)
         payload = _build_chatgpt_responses_payload(model, messages_copy, temperature, max_tokens, stream=True)
     else:
-        target_url = url
+        target_url = _normalize_openai_chat_url(url)
         payload = {
             "model": model,
             "messages": messages_copy,
@@ -1905,6 +1931,8 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
             payload[tok_key] = max_tokens
         if tools:
             payload["tools"] = tools
+        elif tool_choice_none:
+            payload["tool_choice"] = "none"
         # Mistral thinking-capable models — send reasoning_effort so Mistral
         # activates thinking mode and returns structured reasoning_content.
         # Effort level is configurable via ODYSSEUS_MISTRAL_REASONING_EFFORT
@@ -2301,6 +2329,9 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
                                         if reasoning:
                                             yield _stream_delta_event(reasoning, thinking=True)
                                         if content:
+                                            content = _strip_visible_chat_template_artifacts(content)
+                                            if not content:
+                                                continue
                                             content = re.sub(r"<mm:think(\s+[^>]*)?>", r"<think\1>", content, flags=re.IGNORECASE)
                                             content = re.sub(r"</mm:think>", "</think>", content, flags=re.IGNORECASE)
                                             stripped = content.lstrip()

src/model_context.py

@@ -316,6 +316,83 @@ def _lookup_known(model: str) -> Optional[int]:
     return best_ctx
 
 
+def _model_ctx_from_entry(m: dict) -> Optional[int]:
+    """Extract a positive context window from one /models catalog entry.
+
+    Checks the common top-level fields first, then a nested meta/model_extra
+    object. Returns None when no positive window is reported.
+    """
+    if not isinstance(m, dict):
+        return None
+    for field in (
+        "context_length",
+        "context_window",
+        "max_model_len",
+        "max_context_length",
+        "max_seq_len",
+    ):
+        val = m.get(field)
+        if val and isinstance(val, (int, float)) and val > 0:
+            return int(val)
+    meta = m.get("meta") or m.get("model_extra") or {}
+    if isinstance(meta, dict):
+        # n_ctx is the actual serving context (set via -c flag in llama.cpp)
+        for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
+            val = meta.get(field)
+            if val and isinstance(val, (int, float)) and val > 0:
+                return int(val)
+    return None
+
+
+# Per-endpoint cache of the {model_id: context_length} map parsed from a
+# proxy/api catalog. api/proxy endpoints skip the /models download on every
+# lookup because a large catalog is expensive; caching the whole map lets us
+# pay that download at most once per endpoint instead of once per model.
+_catalog_ctx_cache: Dict[str, Dict[str, int]] = {}
+
+
+def _proxy_catalog_context(endpoint_url: str, model: str) -> Optional[int]:
+    """Context window for a model read from the endpoint's /models catalog.
+
+    Fetches the catalog once per endpoint and caches the full id->context map,
+    so an api/proxy endpoint serving a model that isn't in KNOWN_CONTEXT_WINDOWS
+    (e.g. a new OpenRouter model) still reports its real window instead of the
+    bare default. Returns None when the catalog can't be read or doesn't list a
+    positive window for the model.
+    """
+    cat = _catalog_ctx_cache.get(endpoint_url)
+    if cat is None:
+        from src.endpoint_resolver import build_models_url
+        try:
+            r = httpx.get(build_models_url(endpoint_url), timeout=REQUEST_TIMEOUT)
+        except Exception as e:
+            logger.debug(f"Failed to fetch proxy catalog for context length: {e}")
+            return None
+        if not r.is_success:
+            return None
+        cat = {}
+        try:
+            for m in (r.json().get("data") or []):
+                mid = m.get("id") if isinstance(m, dict) else None
+                ctx = _model_ctx_from_entry(m) if mid else None
+                if mid and ctx:
+                    cat[mid] = ctx
+        except Exception as e:
+            logger.debug(f"Failed to parse proxy catalog for context length: {e}")
+            return None
+        _catalog_ctx_cache[endpoint_url] = cat
+
+    if model in cat:
+        return cat[model]
+    # Catalog ids may carry a provider prefix (e.g. "openai/gpt-4o") while the
+    # session stores the bare id; match on the trailing segment as a fallback.
+    base = model.split("/")[-1]
+    for mid, ctx in cat.items():
+        if mid.split("/")[-1] == base:
+            return ctx
+    return None
+
+
 def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
     """Query the model API for context length. Returns (context_length, known) where
     ``known`` is False only for the bare DEFAULT_CONTEXT fallback."""
@@ -330,6 +407,14 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
         if known:
             logger.info(f"Using known context window for {model}: {known}")
             return known, True
+        # Not in the known table: read the real window from the catalog (cached
+        # once per endpoint) instead of capping every unknown model at the
+        # default — that under-reported large windows on aggregators like
+        # OpenRouter (issue #4886).
+        api_ctx = _proxy_catalog_context(endpoint_url, model)
+        if api_ctx:
+            logger.info(f"Proxy catalog reports context window for {model}: {api_ctx}")
+            return api_ctx, True
         return DEFAULT_CONTEXT, False
 
     # Try llama.cpp /slots endpoint first — reports actual serving context
@@ -370,27 +455,7 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
             for m in models_list:
                 mid = m.get("id", "")
                 if mid == model or mid.split("/")[-1] == model.split("/")[-1]:
-                    for field in (
-                        "context_length",
-                        "context_window",
-                        "max_model_len",
-                        "max_context_length",
-                        "max_seq_len",
-                    ):
-                        val = m.get(field)
-                        if val and isinstance(val, (int, float)) and val > 0:
-                            api_ctx = int(val)
-                            break
-
-                    if not api_ctx:
-                        meta = m.get("meta") or m.get("model_extra") or {}
-                        if isinstance(meta, dict):
-                            # n_ctx is the actual serving context (set via -c flag in llama.cpp)
-                            for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
-                                val = meta.get(field)
-                                if val and isinstance(val, (int, float)) and val > 0:
-                                    api_ctx = int(val)
-                                    break
+                    api_ctx = _model_ctx_from_entry(m)
                     break
     except Exception as e:
         logger.debug(f"Failed to query context length for {model}: {e}")

src/personal_docs.py

@@ -68,6 +68,8 @@ def read_text_file(path: str) -> str:
 
 def split_chunks(text: str, size: int = config.CHUNK_SIZE, overlap: int = config.CHUNK_OVERLAP) -> List[str]:
     """Split text into overlapping chunks."""
+    if not isinstance(text, str):
+        return []
     text = text.strip()
     if not text:
         return []
@@ -87,7 +89,8 @@ def split_chunks(text: str, size: int = config.CHUNK_SIZE, overlap: int = config
 
 def tokenize(s: str) -> Set[str]:
     """Tokenize string into words, excluding stop words."""
-    tokens = re.findall(r"[A-Za-z0-9_\-]+", (s or "").lower())
+    text = s if isinstance(s, str) else ""
+    tokens = re.findall(r"[A-Za-z0-9_\-]+", text.lower())
     return set(t for t in tokens if t not in config.STOP_WORDS and len(t) > 1)
 
 def load_personal_index(

src/rag_manager.py

@@ -32,9 +32,9 @@ class RAGManager:
         logger.info("RAGManager initialized as wrapper for VectorRAG")
     
     # Delegate all methods to VectorRAG
-    def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
+    def search(self, query: str, k: int = 5, owner: Optional[str] = None) -> List[Dict[str, Any]]:
         """Search for documents - delegates to VectorRAG."""
-        return self.vector_rag.search(query, k)
+        return self.vector_rag.search(query, k, owner=owner)
     
     def index_personal_documents(
         self,

src/session_search.py

@@ -207,6 +207,7 @@ def _search_like(
     )
     if not include_archived:
         q = q.filter(DBSession.archived == False)
+    q = q.filter(~DBSession.name.like("SFT trace batch%"))
     if restrict_owner:
         q = _owner_filter(q, owner, include_legacy_owner)
     rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
@@ -270,6 +271,7 @@ def _search_fts(
         WHERE chat_messages_fts MATCH :fts_query
           {archived_clause}
           {owner_clause}
+          AND s.name NOT LIKE 'SFT trace batch%'
           AND m.role IN ('user', 'assistant')
         ORDER BY bm25(chat_messages_fts), m.timestamp DESC
         LIMIT :limit

src/settings.py

@@ -136,6 +136,10 @@ DEFAULT_SETTINGS = {
     "task_model": "",
     "default_endpoint_id": "",
     "default_model": "",
+    # Optional prose style used only for normal document writing/editing.
+    # Email replies use email_writing_style instead because greetings,
+    # signatures, and mailbox identity rules are medium-specific.
+    "document_writing_style": "",
     # Ordered fallback chain for the default chat model. Each entry is
     # {"endpoint_id": "...", "model": "..."}. If the primary model fails
     # before producing output (endpoint offline / errors), the chat

src/task_endpoint.py

@@ -6,6 +6,7 @@ from src.endpoint_resolver import (
     resolve_utility_fallback_candidates,
 )
 from src.llm_core import llm_call_async_with_fallback
+from src.interactive_gate import wait_for_interactive_quiet
 
 
 def resolve_task_endpoint(fallback_url=None, fallback_model=None, fallback_headers=None, owner=None):
@@ -72,4 +73,5 @@ async def task_llm_call_async(
     )
     if not candidates:
         raise RuntimeError("No LLM endpoint available for background task")
+    await wait_for_interactive_quiet("background task LLM")
     return await llm_call_async_with_fallback(candidates, messages=messages, **kwargs)

src/task_scheduler.py

@@ -239,6 +239,7 @@ HOUSEKEEPING_DEFAULTS = {
     "tidy_research":        {"name": "Research Tidy",            "trigger_type": "event", "trigger_event": "research_completed", "trigger_count": 5, "schedule": None, "scheduled_time": None, "cron_expression": None, "legacy_names": ["Tidy Research"]},
     "summarize_emails":     {"name": "Email (Summary)",          "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */2 * * *", "ship_paused": True, "legacy_names": ["Tidy Email (Summary)"]},
     "draft_email_replies":  {"name": "Email AI Auto Reply",      "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */2 * * *", "ship_paused": True, "legacy_names": ["Tidy Email (Replies)", "AI Auto Reply"]},
+    "email_auto_translate": {"name": "Email Auto Translate",     "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */2 * * *", "ship_paused": True, "legacy_names": ["Auto-translate Emails", "Auto Translate Email"]},
     "extract_email_events": {"name": "Email Calendar Events",    "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */1 * * *", "ship_paused": True, "legacy_names": ["Email → Calendar Events"]},
     "classify_events":      {"name": "Calendar Classify Events", "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 6,18 * * *", "ship_paused": True, "legacy_names": ["Classify Calendar Events"]},
     "check_email_urgency":   {"name": "Email Tags",               "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 * * * *", "ship_paused": True, "old_cron_expressions": ["*/15 * * * *"], "legacy_names": ["Email Triage", "Urgent Email"]},
@@ -686,6 +687,12 @@ class TaskScheduler:
         db = SessionLocal()
         try:
             now = _utcnow()
+            foreground_active = False
+            try:
+                from src.interactive_gate import has_foreground_activity
+                foreground_active = has_foreground_activity()
+            except Exception:
+                foreground_active = False
             async with self._executing_lock:
                 # Snapshot under the lock so we don't race with mid-iteration adds.
                 executing_snapshot = set(self._executing)
@@ -699,8 +706,13 @@ class TaskScheduler:
                 for task in due:
                     if task.id in self._executing:
                         continue
+                    if foreground_active:
+                        task.next_run = now + timedelta(minutes=15)
+                        continue
                     self._executing.add(task.id)
                     to_dispatch.append(task.id)
+                if foreground_active and due:
+                    db.commit()
             for task_id in to_dispatch:
                 asyncio.create_task(self._execute_task(task_id))
         finally:
@@ -734,15 +746,26 @@ class TaskScheduler:
 
         try:
             if bypass_model_slot or not self._task_needs_model_slot(task_id):
-                await self._execute_task_locked(task_id, run_id, release_executing=release_executing)
+                await self._execute_task_locked(
+                    task_id,
+                    run_id,
+                    release_executing=release_executing,
+                    gate_foreground=not bypass_model_slot,
+                )
                 return
 
             async with self._run_semaphore:
-                await self._execute_task_locked(task_id, run_id, release_executing=release_executing)
+                await self._execute_task_locked(
+                    task_id,
+                    run_id,
+                    release_executing=release_executing,
+                    gate_foreground=True,
+                )
         except asyncio.CancelledError:
             # If cancellation happens while queued behind the semaphore,
             # _execute_task_locked never runs and cannot update the Activity row.
             self._mark_run_aborted(task_id, run_id)
+            self._defer_immediately_due_task(task_id, delay=timedelta(minutes=15))
             raise
         finally:
             handle = self._task_handles.get(task_id)
@@ -752,7 +775,36 @@ class TaskScheduler:
                 async with self._executing_lock:
                     self._executing.discard(task_id)
 
-    async def _execute_task_locked(self, task_id: str, run_id: str, *, release_executing: bool = True):
+    def _defer_immediately_due_task(self, task_id: str, *, delay: timedelta):
+        """A queued task can be cancelled before _execute_task_locked gets a DB
+        handle. If its next_run stays in the past, the scheduler dispatches it
+        again on the next tick and spams aborted Activity rows."""
+        try:
+            from core.database import SessionLocal, ScheduledTask
+            db = SessionLocal()
+            try:
+                task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
+                if (
+                    task
+                    and task.status == "active"
+                    and task.next_run is not None
+                    and task.next_run <= _utcnow()
+                ):
+                    task.next_run = _utcnow() + delay
+                    db.commit()
+            finally:
+                db.close()
+        except Exception:
+            logger.debug("Failed to defer cancelled queued task %s", task_id, exc_info=True)
+
+    async def _execute_task_locked(
+        self,
+        task_id: str,
+        run_id: str,
+        *,
+        release_executing: bool = True,
+        gate_foreground: bool = True,
+    ):
         from core.database import SessionLocal, ScheduledTask, TaskRun
 
         db = SessionLocal()
@@ -769,6 +821,14 @@ class TaskScheduler:
                     db.commit()
                 return
 
+            if gate_foreground:
+                waiting = db.query(TaskRun).filter(TaskRun.id == run_id).first()
+                if waiting and waiting.status == "queued":
+                    waiting.result = "Queued — waiting for Odysseus to be idle…"
+                    db.commit()
+                from src.interactive_gate import wait_for_interactive_quiet
+                await wait_for_interactive_quiet(f"scheduled task {task.name}")
+
             # Flip the run from queued → running. Reset started_at to the
             # actual execution start so queue wait time is visible from
             # created_at vs started_at if we ever surface that.
@@ -799,6 +859,27 @@ class TaskScheduler:
             # previous llm/research run's model. The executors set it once the
             # model is resolved.
             self._last_run_model = None
+            foreground_cancel = {"hit": False}
+            foreground_monitor = None
+            if gate_foreground:
+                current_task = asyncio.current_task()
+
+                async def _cancel_if_foreground_active():
+                    # Give the just-finished quiet gate a tiny grace window,
+                    # then keep enforcing "background means background" while
+                    # a long email/LLM action is already running.
+                    await asyncio.sleep(1.0)
+                    from src.interactive_gate import has_foreground_activity
+                    while True:
+                        await asyncio.sleep(1.0)
+                        if has_foreground_activity():
+                            foreground_cancel["hit"] = True
+                            logger.info("Task '%s' interrupted because Odysseus became active", task.name)
+                            if current_task:
+                                current_task.cancel()
+                            return
+
+                foreground_monitor = asyncio.create_task(_cancel_if_foreground_active())
             try:
                 if task_type == "action":
                     result, success = await self._execute_action(task, run_id=run_id)
@@ -838,15 +919,22 @@ class TaskScheduler:
                 db.commit()
                 return
             except asyncio.CancelledError:
-                logger.info("Task '%s' stopped by user", task.name)
+                msg = (
+                    "Paused because Odysseus became active"
+                    if foreground_cancel.get("hit")
+                    else "Stopped by user"
+                )
+                logger.info("Task '%s' %s", task.name, msg)
                 run_obj = db.query(TaskRun).filter(TaskRun.id == run_id).first()
                 if run_obj:
                     run_obj.status = "aborted"
-                    run_obj.error = "Stopped by user"
-                    run_obj.result = run_obj.result or "Stopped by user"
+                    run_obj.error = msg
+                    run_obj.result = run_obj.result or msg
                     run_obj.finished_at = _utcnow()
                 task.last_run = _utcnow()
-                if (task.trigger_type or "schedule") == "schedule":
+                if foreground_cancel.get("hit"):
+                    task.next_run = _utcnow() + timedelta(minutes=15)
+                elif (task.trigger_type or "schedule") == "schedule":
                     task.next_run = compute_next_run(
                         task.schedule, task.scheduled_time,
                         task.scheduled_day, task.scheduled_date,
@@ -881,6 +969,13 @@ class TaskScheduler:
                     task.next_run = None
                 db.commit()
                 return
+            finally:
+                if foreground_monitor and not foreground_monitor.done():
+                    foreground_monitor.cancel()
+                    try:
+                        await foreground_monitor
+                    except asyncio.CancelledError:
+                        pass
 
             run.finished_at = _utcnow()
 
@@ -1050,6 +1145,7 @@ class TaskScheduler:
         "learn_sender_signatures",
         "summarize_emails",
         "draft_email_replies",
+        "email_auto_translate",
         "extract_email_events",
         "classify_events",
         "tidy_sessions",
@@ -1063,6 +1159,7 @@ class TaskScheduler:
     _MODEL_BACKED_ACTIONS = frozenset({
         "summarize_emails",
         "draft_email_replies",
+        "email_auto_translate",
         "extract_email_events",
         "classify_events",
         "learn_sender_signatures",
@@ -1119,6 +1216,8 @@ class TaskScheduler:
                 self._set_run_progress(run_id, message)
 
             kwargs = {"owner": task.owner, "task_name": task.name, "progress_cb": _progress}
+            if task.prompt:
+                kwargs["prompt"] = task.prompt
             if task.action in ("run_script", "run_local", "ssh_command") and task.prompt:
                 kwargs["script" if task.action in ("run_script", "run_local") else "command"] = task.prompt
             # cookbook_serve carries its JSON config in task.prompt — feed it
@@ -1682,8 +1781,15 @@ class TaskScheduler:
 
         target = (output or "").strip()
         explicit = ""
+        account_id = ""
         if target.startswith("email:"):
             explicit = target.split(":", 1)[1].strip()
+            if "|account=" in explicit:
+                explicit, account_id = explicit.split("|account=", 1)
+                explicit = explicit.strip()
+                account_id = account_id.strip()
+            if explicit == "self":
+                explicit = ""
         elif "@" in target:
             explicit = target
 
@@ -1691,7 +1797,7 @@ class TaskScheduler:
             from routes.email_routes import _resolve_send_config
             from routes.email_helpers import _send_smtp_message
 
-            cfg = _resolve_send_config(owner=task.owner or "")
+            cfg = _resolve_send_config(account_id=account_id or None, owner=task.owner or "")
             to_addr = explicit or cfg.get("from_address") or cfg.get("smtp_user") or ""
             if not to_addr:
                 raise RuntimeError("No email recipient resolved for task output")
@@ -1759,6 +1865,8 @@ class TaskScheduler:
         # behind the primary endpoint so a downed primary won't silently yield
         # `(no output)`.
         try:
+            from src.interactive_gate import wait_for_interactive_quiet
+            await wait_for_interactive_quiet(f"agent task {task.name}")
             from src.task_endpoint import resolve_task_candidates
             _task_fallbacks = resolve_task_candidates(
                 fallback_url=endpoint_url,

src/tool_execution.py

@@ -21,7 +21,12 @@ from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
 
 
 
-from src.tool_security import is_public_blocked_tool, owner_is_admin_or_single_user
+from src.tool_security import (
+    BUILTIN_EMAIL_TOOLS,
+    email_tool_policy_names,
+    is_public_blocked_tool,
+    owner_is_admin_or_single_user,
+)
 from src.tool_policy import ToolPolicy
 from src.constants import MAX_OUTPUT_CHARS, MAX_READ_CHARS, MAX_DIFF_LINES, DATA_DIR
 from src.tool_utils import _truncate, get_mcp_manager
@@ -66,25 +71,35 @@ _SENSITIVE_FILE_PATTERNS: tuple[str, ...] = (
     "known_hosts",
 )
 
+# Case-folded views used for matching. On a case-insensitive filesystem
+# (Windows, default macOS) ".SSH/AUTHORIZED_KEYS" and ".env" resolve to the
+# same protected files as their lowercase forms, so the deny-list has to fold
+# case before comparing — the sibling resolver already normcases paths for the
+# same reason. casefold (not os.path.normcase) because normcase is a no-op on
+# POSIX, which is exactly where the macOS read-exfil path lives.
+_SENSITIVE_BASENAMES_CF: frozenset[str] = frozenset(b.casefold() for b in _SENSITIVE_BASENAMES)
+_SENSITIVE_FILE_PATTERNS_CF: frozenset[str] = frozenset(p.casefold() for p in _SENSITIVE_FILE_PATTERNS)
+
 
 def _is_sensitive_path(resolved: str) -> bool:
     """Return True if *resolved* falls under a sensitive directory or
     matches a sensitive filename — regardless of what root it sits under.
+
+    Matching is case-insensitive: on Windows / default macOS a case-variant
+    name (``.SSH``, ``AUTHORIZED_KEYS``, ``Id_Rsa``) points at the same file as
+    the lowercase form, so a case-sensitive check would let it slip past the
+    deny-list in every file tool that relies on it.
     """
-    parts = resolved.split(os.sep)
-    filenames: set[str] = {parts[-1]} if parts else set()
+    parts = [p.casefold() for p in resolved.split(os.sep)]
+    filename = parts[-1] if parts else ""
 
     # Check if any path component is a sensitive directory.
     for part in parts:
-        if part in _SENSITIVE_BASENAMES:
+        if part in _SENSITIVE_BASENAMES_CF:
             return True
 
     # Check filename against known sensitive files.
-    for pat in _SENSITIVE_FILE_PATTERNS:
-        if pat in filenames:
-            return True
-
-    return False
+    return filename in _SENSITIVE_FILE_PATTERNS_CF
 
 
 def _tool_path_roots() -> list[str]:
@@ -390,8 +405,42 @@ _MCP_ARG_PARSERS: Dict[str, Callable[[str], Dict[str, str]]] = {
 }
 
 
+# Primary argument key(s) for the legacy line-parsed tools. When a fenced
+# block's content is a JSON object carrying one of these keys, it's structured
+# inline args (the relaxed parser's ```web_search {"query": "..."}``` shape) —
+# use the object directly instead of letting the line-based parsers wrap the
+# whole JSON string as the query/url/path/prompt. Keyed off membership only
+# (the primary key never changes), so this can't drift; an unrecognized object
+# safely falls through to the line-based parser, i.e. the previous behavior.
+#
+# IMPORTANT — this only covers the MCP path. _build_mcp_args is reached via
+# _call_mcp_tool only for _MCP_TOOL_MAP tools (so an entry outside that map is
+# dead, as manage_memory was). And of these, only generate_image has a live MCP
+# server today; web_search/web_fetch/read_file/write_file have none, so they run
+# via _direct_fallback -> TOOL_HANDLERS, whose handlers decode JSON themselves
+# (see ReadFileTool/WriteFileTool/WebSearchTool/WebFetchTool). The entries here
+# are kept as defense-in-depth for if/when those servers are added. The live
+# fix for each server-less tool lives in its handler. test_write_file_inline_
+# json_args and test_mcp_json_primary_keys_are_all_live pin both halves.
+_MCP_JSON_PRIMARY_KEYS: Dict[str, tuple] = {
+    "web_search":     ("query", "queries"),
+    "web_fetch":      ("url",),
+    "read_file":      ("path",),
+    "write_file":     ("path",),
+    "generate_image": ("prompt",),
+}
+
+
 def _build_mcp_args(tool: str, content: str) -> Dict:
     """Convert fenced-block text content to structured MCP arguments."""
+    primaries = _MCP_JSON_PRIMARY_KEYS.get(tool)
+    if primaries and content.strip().startswith("{"):
+        try:
+            decoded = json.loads(content.strip())
+        except (json.JSONDecodeError, TypeError):
+            decoded = None
+        if isinstance(decoded, dict) and any(k in decoded for k in primaries):
+            return decoded
     parser = _MCP_ARG_PARSERS.get(tool)
     return parser(content) if parser else {}
 
@@ -596,6 +645,12 @@ async def _execute_tool_block_impl(
     tool = block.tool_type
     content = block.content
 
+    # The block/disable gates below must match every policy-equivalent
+    # spelling of the tool name (bare email names alias their mcp__email__
+    # form — see email_tool_policy_names), not just the spelling the model
+    # happened to emit.
+    policy_names = email_tool_policy_names(tool)
+
     # Misformatted tool call detection: model put JSON inside ```python``` (or
     # similar) without naming the tool. Common with MiniMax-style outputs.
     # Return a helpful error so the model retries with the correct format.
@@ -623,13 +678,13 @@ async def _execute_tool_block_impl(
             pass
 
     # Reject tools that the user has disabled for this request
-    if disabled_tools and tool in disabled_tools:
+    if disabled_tools and not policy_names.isdisjoint(disabled_tools):
         desc = f"{tool}: BLOCKED"
         result = {"error": f"Tool '{tool}' is disabled by user.", "exit_code": 1}
         logger.info(f"Tool blocked by user: {tool}")
         return desc, result
 
-    if tool_policy and tool_policy.blocks(tool):
+    if tool_policy and any(tool_policy.blocks(name) for name in policy_names):
         desc = f"{tool}: BLOCKED"
         result = {
             "error": f"Execution of tool '{tool}' is forbade by the active guide-only policy.",
@@ -823,6 +878,51 @@ async def _execute_tool_block_impl(
     elif tool == "vault_unlock":
         desc = "vault_unlock"
         result = await do_vault_unlock(content, owner=owner)
+    elif tool in BUILTIN_EMAIL_TOOLS:
+        # Bare email tool name from fenced-block models (e.g. Ollama) — route to MCP email server.
+        # Non-admin owners never reach here: BUILTIN_EMAIL_TOOLS ⊆ NON_ADMIN_BLOCKED_TOOLS,
+        # so is_public_blocked_tool() above already rejected them.
+        mcp = get_mcp_manager()
+        qualified = f"mcp__email__{tool}"
+        desc = f"email: {tool}"
+        if mcp:
+            _raw = content.strip()
+            args = {}
+            _args_error = None
+            if _raw:
+                # A non-empty body is always meant to be the call's arguments,
+                # and every email tool takes a JSON object. Anything that
+                # isn't one is a correctable error — NOT a silent empty-args
+                # call, which would read the DEFAULT mailbox/folder instead of
+                # the one the model meant (#3966 class). Only an EMPTY body
+                # keeps the no-arg path (e.g. ```list_email_accounts```).
+                try:
+                    parsed = json.loads(_raw)
+                except (json.JSONDecodeError, TypeError) as _je:
+                    # Covers both `{account: "work"}` (looks like JSON, bad)
+                    # and `account: work` (not JSON at all).
+                    _args_error = (
+                        f"'{tool}' arguments are not valid JSON ({_je}). "
+                        'Send a JSON object, e.g. {"account": "work"} — '
+                        "keys and string values need double quotes."
+                    )
+                else:
+                    if isinstance(parsed, dict):
+                        args = parsed
+                    else:
+                        _args_error = (
+                            f"'{tool}' arguments must be a JSON object, "
+                            'e.g. {"uid": "..."} — got a JSON array/value instead.'
+                        )
+            if _args_error is not None:
+                result = {"error": _args_error, "exit_code": 1}
+            else:
+                if owner:
+                    args = dict(args)
+                    args[_EMAIL_MCP_OWNER_ARG] = owner
+                result = await mcp.call_tool(qualified, args)
+        else:
+            result = {"error": "MCP manager not available", "exit_code": 1}
     elif tool.startswith("mcp__"):
         # MCP tool dispatch
         mcp = get_mcp_manager()
@@ -840,12 +940,12 @@ async def _execute_tool_block_impl(
             desc = f"mcp: {tool}"
             result = {"error": "MCP manager not available", "exit_code": 1}
 
-    
+
     elif tool in dynamic_handlers:
         first_line = content.split(chr(10))[0][:80]
         desc = f"registry: {tool} {first_line}".strip()
         res = await _direct_fallback(tool, content, progress_cb=progress_cb)
-        
+
         if isinstance(res, tuple):
             desc, result = res
         else:

src/tool_implementations.py

@@ -36,6 +36,8 @@ def __getattr__(name):
         from src.agent_tools import admin_tools
         return getattr(admin_tools, name)
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
 # Cookbook (model serving) domain extracted to src/tools/cookbook.py
 # (slice 1, #4082/#4071). Re-imported here so this module stays a working
 # facade. cookbook.py pulls `_internal_headers` / `_INTERNAL_BASE` back

src/tool_index.py

@@ -105,12 +105,12 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
     "search_chats": "Search past session transcripts across chats.",
     "ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Omit `multi`/keep it false unless the question explicitly permits choosing multiple options. Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.",
     "update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.",
-    "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply` to open an email reply draft document without sending. To pre-fill the reply body in one shot (USE THIS whenever the user told you what to say — opening an empty draft when they asked you to write is wrong), append the body after the mode: `open_email_reply <uid> <folder> reply <body text>`. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.",
+    "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply <body text>` (or structured body) to open an email reply draft document without sending. USE THIS whenever the user says to write/draft a reply or tells you what to say — opening an empty draft or sending immediately is wrong. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.",
     "list_email_accounts": "List configured email accounts and default status. Use before reading or sending mail when the user mentions Gmail, work mail, custom domain mail, another mailbox, or asks to compare/check multiple inboxes.",
     "list_emails": "List emails for a folder/account, newest first, including read messages by default. Shows subject, sender, date, UID, account, and AI summary. Check inbox, find emails needing replies. Supports account from list_email_accounts for Gmail/work/custom mailboxes. For last/latest/newest email, use max_results=1 and unread_only=false.",
     "read_email": "Read the full content of a specific email by UID or Message-ID. View email body, check details. Supports account from list_email_accounts when the UID belongs to a non-default mailbox.",
     "send_email": "Send a new email via SMTP. Provide recipient, subject, body, and optional account from list_email_accounts. For replying to a thread use reply_to_email instead.",
-    "reply_to_email": "SEND a reply email immediately by UID. Do not use for open/start reply draft requests; use ui_control open_email_reply for those. For follow-up 'reply ...' send requests, use the exact UID and account from latest read_email/list_emails output; never invent UID 1. Threads automatically with In-Reply-To/References, prefixes Re:, marks original as Answered.",
+    "reply_to_email": "SEND a reply email immediately by UID. Do not use for write/draft/open/start reply requests; use ui_control open_email_reply with body so the user can review. Only use when the user explicitly says to send now. For send requests, use the exact UID and account from latest read_email/list_emails output; never invent UID 1. Threads automatically with In-Reply-To/References, prefixes Re:, marks original as Answered.",
     "archive_email": "Move an email out of the inbox into the Archive folder. Use after handling messages you want to keep but get out of the way.",
     "delete_email": "Delete an email — moves to Trash by default, or expunges permanently with permanent=true.",
     "mark_email_read": "Mark an email as read or unread by toggling the \\Seen flag.",
@@ -118,7 +118,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
     "resolve_contact": "Look up a contact's email address by name. Searches CardDAV address book and sent email history. Use when the user says 'message [name]', 'email [name]', or 'send to [name]' without an email address.",
     "manage_contact": "Save / update / delete / list address-book contacts (CardDAV). Use for info about ANOTHER person — name, email, phone, postal address. Args: action=list|add|update|delete, name, email, phones, address, uid (from list). For 'save this for <person>' / address pastes / phone numbers next to a name, this is the right tool — NOT manage_memory. Do NOT use for facts about the USER ('my name is X'); those are manage_memory.",
     "manage_notes": "Create and manage notes and checklists (Google Keep-style). ALWAYS use this for note/todo/checklist/reminder creation — NEVER hit /api/notes via app_api. Accepts natural-language `due_date` like 'tomorrow at 9am' or '11pm today' (parsed in the USER'S timezone). The due_date IS the reminder — it fires a notification at that time, so do NOT also create a calendar event for the same reminder. Set colors, labels, pin, archive. Do NOT use manage_memory for note content.",
-    "manage_calendar": "Calendar event management: list, create, update, delete. Each event can carry a tag/category (event_type — work/personal/health/travel/meal/social/admin/other) and importance (low/normal/high/critical). Resolve today/tomorrow using the Current date and time context, then use ISO datetimes in the user's local wall time; supports all-day events. For event reminders/alarms, pass reminder_minutes; this creates the Notes reminder, so do not also call manage_notes for the same reminder.",
+    "manage_calendar": "Calendar event management: list, create, update, delete. Each event can carry a tag/category (event_type — work/personal/health/travel/meal/social/admin/other) and importance (low/normal/high/critical). Resolve today/tomorrow using the Current date and time context, then use ISO datetimes in the user's local wall time; supports all-day events. Use rrule only for explicit recurrence; for update_event pass rrule='' to remove repeats. For event reminders/alarms, pass reminder_minutes; this creates the Notes reminder, so do not also call manage_notes for the same reminder.",
     "download_model": "Download a HuggingFace model to a local or remote server. Specify repo_id (e.g. 'Qwen/Qwen3-8B'), optional server host, and optional include filter for specific files.",
     "serve_model": "Start serving a model with vLLM, SGLang, llama.cpp, Ollama, or Diffusers. cmd MUST start with the binary directly — e.g. `vllm serve /mnt/HADES/models/Qwen3.5-397B-A17B-AWQ --port 8003 --tensor-parallel-size 8 …`. NEVER prefix with `cd …`, `source …`, or chain with `&&`/`||` — those get rejected by the validator. The venv activation (env_prefix) and CUDA env are added automatically from the target host's saved settings. For image/inpainting/diffusion use python3 scripts/diffusion_server.py --model <repo> --port 8100. After launch, call list_served_models for readiness/errors and retry suggestions. If serve_model fails with 'Invalid characters in cmd', simplify to the bare binary + args.",
     "list_served_models": "List currently running model servers in the Cookbook — shows status (loading, ready, idle, error), model name, port, throughput, and serve failure diagnosis/retry suggestions. Use when the user asks 'what's running', 'show my cookbook', 'which models are up', 'what's serving'.",

src/tool_parsing.py

@@ -10,9 +10,10 @@ import bisect
 import json
 import logging
 import re
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 from src.agent_tools import ToolBlock, TOOL_TAGS
+from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
@@ -20,12 +21,63 @@ logger = logging.getLogger(__name__)
 # Regex patterns
 # ---------------------------------------------------------------------------
 
-# Pattern 1: ```bash ... ``` fenced code blocks
+# Pattern 1: ```bash ... ``` fenced code blocks. The tag may be followed by a
+# newline (classic form) or by inline JSON args on the same line
+# (```list_email_accounts {}). The same-line part is captured separately
+# (group 2) and judged by _fenced_tool_call below — the regex alone only
+# requires it to start with { or [; anything else after the tag is a Markdown
+# info string (```python title="example.py") and the fence never matches.
+# (?![\w-]) keeps the alternation from prefix-matching longer fence tags:
+# without it, ```python3 would match as tool "python" with content "3\n..."
+# and execute as code.
 _TOOL_BLOCK_RE = re.compile(
-    r"```(" + "|".join(TOOL_TAGS) + r")\s*\n([\s\S]*?)```",
+    r"```(" + "|".join(TOOL_TAGS) + r")(?![\w-])"
+    r"[ \t]*([{\[][^\n]*?)?[ \t]*(?=\r?\n|```)\r?\n?([\s\S]*?)```",
     re.IGNORECASE,
 )
 
+# Tags whose fenced content is raw code, not JSON args. Same-line text after
+# these tags is Markdown fence metadata on a real language (```bash {title=
+# "setup"}), never inline tool args — only the classic tag-then-newline form
+# executes for them.
+_CODE_FENCE_TAGS = frozenset({"bash", "python"})
+
+
+def _fenced_tool_call(m) -> Optional[Tuple[str, str]]:
+    """Classify a Pattern-1 fence match: (tag, content) when it is an
+    executable tool call, None when the fence must stay display text.
+
+    Shared by parse_tool_blocks and strip_tool_blocks so the execute and
+    display decisions can never disagree: a fence that doesn't execute is
+    never stripped, and vice versa.
+
+    Same-line text after the tag only counts as inline tool args when the
+    tag's tool takes JSON args (not a code tag) AND the text is valid
+    standalone JSON. ```bash {title="setup"} and ```python {"x": 1} are
+    fence attributes on real languages, and {title="x"} on any tag is
+    metadata, not arguments — all of those stay visible and inert.
+    """
+    tag = m.group(1).lower()
+    inline = (m.group(2) or "").strip()
+    body = (m.group(3) or "").strip()
+    if not inline:
+        return tag, body
+    if tag in _CODE_FENCE_TAGS:
+        return None
+    # Inline args may continue onto following lines (a JSON object opened on
+    # the tag line); the combined text must parse as JSON or nothing runs.
+    content = f"{inline}\n{body}" if body else inline
+    try:
+        json.loads(content)
+    except (ValueError, TypeError):
+        return None
+    return tag, content
+
+
+def _strip_executed_fence(m) -> str:
+    """re.sub callback: remove only fences that parse as tool calls."""
+    return "" if _fenced_tool_call(m) is not None else m.group(0)
+
 # Pattern 2: [TOOL_CALL] ... [/TOOL_CALL] blocks (some models use this format)
 # Matches: {tool => "shell", args => {--command "ls -la"}} etc.
 _TOOL_CALL_RE = re.compile(
@@ -114,6 +166,13 @@ _TOOL_CODE_RE = re.compile(
 _TOOL_CODE_OPEN_RE = re.compile(r"<tool_code>\s*\{", re.IGNORECASE)
 _TOOL_CODE_CLOSE_RE = re.compile(r"\}\s*</tool_code>", re.IGNORECASE)
 
+# Pattern 4b: Gemma-style <|tool_call|> call:tool_name{args} <tool_call|>
+_GEMMA_TOOL_CALL_RE = re.compile(
+    r"<\|?tool_call\|?>\s*call:([\w\d_-]+)\s*(\{[\s\S]*?\})\s*<\|?tool_call\|?>",
+    re.IGNORECASE,
+)
+
+
 # Pattern 5: DeepSeek DSML markup leaking into content. When deepseek
 # models can't emit structured tool_calls (e.g. we sent no tool schemas
 # that round, or the API didn't parse them), they fall back to raw
@@ -256,6 +315,17 @@ _RAW_WEB_JSON_TOOL_RE = re.compile(
 )
 _RAW_WEB_JSON_ALLOWED_KEYS = {"query", "queries", "time_filter", "freshness", "max_pages"}
 
+# Narrow rescue for models that ignore native tool calling and print the UI
+# command as plain text. Keep this intentionally tiny: open-panel is a harmless
+# frontend event, while broad plain-text parsing of shell/doc/email tools would
+# be unsafe.
+_PLAIN_UI_OPEN_PANEL_RE = re.compile(
+    r"(?im)^\s*(?:`{1,3})?\s*ui_control\s+open_panel\s+"
+    r"(documents?|library|gallery|images?|email|inbox|mail|sessions?|chats?|history|"
+    r"notes?|brain|memor(?:y|ies)|skills?|settings|preferences|cookbook|models?)"
+    r"\s*(?:`{1,3})?\s*$"
+)
+
 
 # ---------------------------------------------------------------------------
 # Parsing functions
@@ -780,6 +850,40 @@ def _parse_tool_code_block(raw: str) -> Optional[ToolBlock]:
         return ToolBlock(tool_name, content.strip())
     return None
 
+def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
+    """Parse a Gemma-style call:tool_name{...} block into a ToolBlock."""
+    tool_name = tool_name.strip().lower().replace("-", "_")
+    body = body.strip()
+    if not body:
+        return None
+
+    # Replace custom Gemma string delimiters with standard quotes
+    body = body.replace('<|"|>', '"').replace('<|"', '"').replace('"|>', '"')
+
+    # Try standard JSON parsing
+    params = {}
+    try:
+        params = json.loads(body)
+        if not isinstance(params, dict):
+            params = {}
+    except json.JSONDecodeError:
+        # Try unquoted keys repair: e.g. {query: "..."} -> {"query": "..."}
+        try:
+            repaired = re.sub(r'([{,]\s*)(\w+)\s*:', r'\1"\2":', body)
+            params = json.loads(repaired)
+            if not isinstance(params, dict):
+                params = {}
+        except Exception:
+            # Simple regex key-value extraction fallback
+            params = {}
+            for m in re.finditer(r'(\w+)\s*:\s*["\']?(.*?)["\']?(?=\s*,\s*\w+\s*:|\s*\})', body):
+                k = m.group(1)
+                v = m.group(2).strip()
+                params[k] = v
+
+    from src.tool_schemas import function_call_to_tool_block
+    return function_call_to_tool_block(tool_name, json.dumps(params))
+
 
 def _iter_delimited(text, open_re, close_re):
     """Yield ``(match_start, inner_start, inner_end, match_end)`` for each
@@ -923,9 +1027,20 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]:
     # Pattern 1: fenced code blocks (skipped when `skip_fenced` — see docstring).
     if not skip_fenced:
         for m in _TOOL_BLOCK_RE.finditer(text):
-            tag = m.group(1).lower()
-            content = m.group(2).strip()
+            call = _fenced_tool_call(m)
+            if call is None:
+                continue
+            tag, content = call
             if not content:
+                # An empty fence is still an unambiguous call for the email
+                # tools — ```list_email_accounts``` with no body is a shape
+                # local models really emit for no-arg tools. Dispatch with
+                # empty args and let the tool's own validation answer;
+                # silently dropping the call left models concluding email was
+                # broken. Other tags (bash, python, ...) keep skipping: empty
+                # content is nothing to run.
+                if tag in BUILTIN_EMAIL_TOOLS:
+                    blocks.append(ToolBlock(tag, ""))
                 continue
             # If a code block's content is an <invoke> XML call (some models wrap
             # tool calls in ```python or ```xml fences), parse the invoke instead.
@@ -1012,12 +1127,29 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]:
             if block:
                 blocks.append(block)
 
+    # Pattern 4b: Gemma-style <|tool_call|> blocks
+    if not blocks:
+        for m in _GEMMA_TOOL_CALL_RE.finditer(text):
+            tool_name = m.group(1)
+            body = m.group(2)
+            block = _parse_gemma_tool_call(tool_name, body)
+            if block:
+                blocks.append(block)
+
     # Pattern 6: local text-model web_search call leaked as prose + bare JSON.
     if not blocks and not skip_fenced:
         raw_web_json = _parse_raw_web_json_lookup(text)
         if raw_web_json:
             blocks.append(raw_web_json[0])
 
+    # Pattern 7: plain `ui_control open_panel notes` line. This commonly comes
+    # from weaker native-tool models after reading the tool docs but failing to
+    # emit the actual structured call.
+    if not blocks:
+        m = _PLAIN_UI_OPEN_PANEL_RE.search(text)
+        if m:
+            blocks.append(ToolBlock("ui_control", f"open_panel {m.group(1).lower()}"))
+
     return blocks
 
 
@@ -1037,7 +1169,10 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str:
     # Normalize DSML first so its markup gets stripped by the <invoke>
     # / <tool_call> removers below instead of leaking to the user.
     text = _normalize_dsml(text)
-    cleaned = text if skip_fenced else _TOOL_BLOCK_RE.sub('', text)
+    # Keep the executed-vs-illustrative fence distinction (only strip fences
+    # that actually dispatched; leave example fences from native models inert
+    # but visible), then remove [TOOL_CALL]{...}[/TOOL_CALL] markup.
+    cleaned = text if skip_fenced else _TOOL_BLOCK_RE.sub(_strip_executed_fence, text)
     # Forward-only removal mirrors parse_tool_blocks: _strip_delimited pairs each
     # opener with a later closer and stops when none is reachable, so untrusted
     # output can't drive the O(n^2) lazy-rescan (ReDoS); see _iter_delimited.
@@ -1046,11 +1181,13 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str:
     cleaned = _strip_delimited(cleaned, _XML_TOOL_CALL_OPEN_RE, _XML_TOOL_CALL_CLOSE_RE)
     cleaned = _XML_OPEN_TOOL_CALL_RE.sub('', cleaned)
     cleaned = _strip_delimited(cleaned, _TOOL_CODE_OPEN_RE, _TOOL_CODE_CLOSE_RE)
+    cleaned = _GEMMA_TOOL_CALL_RE.sub('', cleaned)
     if not skip_fenced:
         raw_web_json = _parse_raw_web_json_lookup(cleaned)
         if raw_web_json:
             _, (start, end) = raw_web_json
             cleaned = cleaned[:start] + cleaned[end:]
+    cleaned = _PLAIN_UI_OPEN_PANEL_RE.sub("", cleaned)
     # Strip bare <invoke> blocks not wrapped in <tool_call>
     cleaned = _strip_bare_invoke_markup(cleaned)
     cleaned = re.sub(r'\n{3,}', '\n\n', cleaned)

src/tool_schemas.py

@@ -14,6 +14,7 @@ from typing import Optional
 
 from src.agent_tools import ToolBlock, TOOL_TAGS
 from src.tool_parsing import _TOOL_NAME_MAP
+from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
@@ -325,7 +326,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "send_to_session",
-            "description": "Send a message to an existing chat and get the model's response. The chat keeps its conversation history.",
+            "description": "Send a new message to an existing live chat and get that chat model's response. Do not use this to retrieve, read, summarize, or inspect old chats; use search_chats or list_sessions for past chat evidence.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -415,7 +416,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "ui_control",
-            "description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), open_email_reply (open an email reply draft document; does NOT send), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation). When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.",
+            "description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), open_email_reply (open an email reply draft document; DOES NOT send. For 'write/draft a reply saying X', include body with the drafted reply), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation). When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -426,6 +427,7 @@ FUNCTION_TOOL_SCHEMAS = [
                     "uid": {"type": "string", "description": "Email UID for open_email_reply"},
                     "folder": {"type": "string", "description": "Email folder for open_email_reply (default INBOX)"},
                     "mode": {"type": "string", "description": "Reply draft mode for open_email_reply: reply, reply-all, or ai-reply"},
+                    "body": {"type": "string", "description": "For open_email_reply: reply body to pre-fill. Required whenever the user told you what the reply should say. Opens a draft, does not send."},
                     "colors": {"type": "object", "description": "For create_theme: the theme colors",
                                "properties": {
                                    "bg": {"type": "string", "description": "Background color (hex, e.g. #1a1a2e)"},
@@ -538,7 +540,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "manage_calendar",
-            "description": "Manage calendar events: list events in a date range, create, update, delete. Each event can carry a tag/category (event_type) and importance level. Resolve relative dates like today/tomorrow against the 'Current date and time' system context, then pass ISO 8601 datetimes in the user's local wall time; for all-day events set all_day=true and pass YYYY-MM-DD. For event reminders/alarms, pass reminder_minutes; the tool creates the Odysseus note reminder, so do not also call manage_notes for the same reminder.",
+            "description": "Manage calendar events: list events in a date range, create, update, delete. Each event can carry a tag/category (event_type) and importance level. Resolve relative dates like today/tomorrow against the 'Current date and time' system context, then pass ISO 8601 datetimes in the user's local wall time; for all-day events set all_day=true and pass YYYY-MM-DD. For event reminders/alarms, pass reminder_minutes; the tool creates the Odysseus note reminder, so do not also call manage_notes for the same reminder. Do not set rrule for single-occurrence requests such as 'next Wednesday only'; use rrule only when the user explicitly wants recurrence.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -559,7 +561,7 @@ FUNCTION_TOOL_SCHEMAS = [
                     "event_type": {"type": "string", "description": "Tag / category for the event. Common values: work, personal, health, travel, meal, social, admin, other. Aliases accepted: tag, category, type."},
                     "importance": {"type": "string", "enum": ["low", "normal", "high", "critical"], "description": "Priority level (defaults to 'normal')"},
                     "reminder_minutes": {"type": "integer", "description": "For create_event: create an Odysseus reminder this many minutes before the event, e.g. 5 for 'reminder 5 min before'."},
-                    "rrule": {"type": "string", "description": "Recurrence rule in iCalendar RRULE format, e.g. 'FREQ=WEEKLY;BYDAY=MO' for weekly on Monday. Use with create_event or update_event."}
+                    "rrule": {"type": "string", "description": "Recurrence rule in iCalendar RRULE format, e.g. 'FREQ=WEEKLY;BYDAY=MO' for weekly on Monday. Use with create_event or update_event. For update_event, pass an explicit empty string to remove recurrence and make the event single-occurrence."}
                 },
                 "required": ["action"]
             }
@@ -569,12 +571,12 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "manage_notes",
-            "description": "Manage notes and checklists (Google Keep-style): list, add, update, delete, toggle_item. IMPORTANT: For to-do lists / checklists, set note_type='checklist' and pass the items as the `checklist_items` array — do NOT serialize them into `content` as plain text. For freeform notes, use note_type='note' and put the body in `content`. `due_date` accepts natural language like 'tomorrow at 9am' (parsed in the user's timezone) and fires a notification — do not also create a calendar event for the same reminder.",
+            "description": "Manage notes and checklists (Google Keep-style): list, view, add, update, delete, toggle_item. Use list/search to find candidate notes, then view with the note id when you need the full body. IMPORTANT: For to-do lists / checklists, set note_type='checklist' and pass the items as the `checklist_items` array — do NOT serialize them into `content` as plain text. For freeform notes, use note_type='note' and put the body in `content`. `due_date` accepts natural language like 'tomorrow at 9am' (parsed in the user's timezone) and fires a notification — do not also create a calendar event for the same reminder.",
             "parameters": {
                 "type": "object",
                 "properties": {
                     "action": {"type": "string",
-                               "enum": ["list", "add", "update", "delete", "toggle_item"],
+                               "enum": ["list", "view", "add", "update", "delete", "toggle_item"],
                                "description": "The action to perform"},
                     "id": {"type": "string", "description": "Note id (for update/delete/toggle_item); 8-char prefix is fine"},
                     "title": {"type": "string", "description": "Note title (for add/update)"},
@@ -1106,7 +1108,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "reply_to_email",
-            "description": "SEND a reply email immediately by UID. Do not use this when the user asks to open/start a reply window or draft; use ui_control action=open_email_reply instead. For follow-up 'reply ...' requests where the user clearly wants to send now, use the exact UID from the latest read_email/list_emails result; never invent UID 1. Automatically threads with In-Reply-To/References headers.",
+            "description": "SEND a reply email immediately by UID. Do not use this when the user asks to write/draft/open/start a reply; use ui_control action=open_email_reply with body instead so the user can review. Only use when the user explicitly says to send now. Use the exact UID from the latest read_email/list_emails result; never invent UID 1. Automatically threads with In-Reply-To/References headers.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -1210,27 +1212,97 @@ FUNCTION_TOOL_SCHEMAS = [
 # Converter: native function call -> ToolBlock
 # ---------------------------------------------------------------------------
 
+def _decode_loose_json_string(value: str) -> str:
+    """Decode common JSON string escapes without requiring inner quotes to be escaped."""
+    out = []
+    i = 0
+    while i < len(value):
+        ch = value[i]
+        if ch != "\\" or i + 1 >= len(value):
+            out.append(ch)
+            i += 1
+            continue
+        nxt = value[i + 1]
+        if nxt == "n":
+            out.append("\n")
+        elif nxt == "r":
+            out.append("\r")
+        elif nxt == "t":
+            out.append("\t")
+        elif nxt == "b":
+            out.append("\b")
+        elif nxt == "f":
+            out.append("\f")
+        elif nxt in ('"', "\\", "/"):
+            out.append(nxt)
+        elif nxt == "u" and i + 5 < len(value):
+            try:
+                out.append(chr(int(value[i + 2:i + 6], 16)))
+                i += 4
+            except ValueError:
+                out.append("\\" + nxt)
+        else:
+            out.append("\\" + nxt)
+        i += 2
+    return "".join(out)
+
+
+def _repair_document_function_args(tool_type: str, arguments: str) -> Optional[dict]:
+    """Salvage obvious malformed document tool args from local model wrappers.
+
+    The doc LoRA sometimes emits the right native tool call but puts raw quotes
+    inside the document text, making the surrounding JSON invalid. Treat that as
+    a wrapper parse failure, not a semantic tool-choice failure.
+    """
+    if tool_type != "update_document" or not isinstance(arguments, str):
+        return None
+    raw = arguments.strip()
+    if not raw.startswith("{") or not raw.endswith("}"):
+        return None
+    for key in ("content", "conten"):
+        marker = f'"{key}"'
+        key_pos = raw.find(marker)
+        if key_pos < 0:
+            continue
+        colon_pos = raw.find(":", key_pos + len(marker))
+        if colon_pos < 0:
+            continue
+        first_quote = raw.find('"', colon_pos + 1)
+        if first_quote < 0:
+            continue
+        close_brace = raw.rfind("}")
+        last_quote = raw.rfind('"', first_quote + 1, close_brace)
+        if last_quote <= first_quote:
+            continue
+        content = _decode_loose_json_string(raw[first_quote + 1:last_quote])
+        return {"content": content}
+    return None
+
+
 def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock]:
     """Convert a native function call into a ToolBlock for the existing execution pipeline."""
+    tool_type = _TOOL_NAME_MAP.get(name, name)
     try:
         if not arguments or (isinstance(arguments, str) and not arguments.strip()):
             args = {}
         else:
             args = json.loads(arguments) if isinstance(arguments, str) else arguments
     except (json.JSONDecodeError, TypeError):
-        logger.error(f"Failed to parse function call arguments for {name}: {arguments}")
-        return None
-
-    tool_type = _TOOL_NAME_MAP.get(name, name)
-    _BUILTIN_EMAIL_TOOLS = {"list_email_accounts", "send_email", "list_emails", "read_email", "reply_to_email",
-                            "archive_email", "delete_email", "mark_email_read", "bulk_email", "download_attachment"}
+        args = _repair_document_function_args(tool_type, arguments)
+        if args is not None:
+            logger.warning(f"Repaired malformed document function call arguments for {name}")
+        else:
+            logger.error(f"Failed to parse function call arguments for {name}: {arguments}")
+            return None
 
     # Some models emit valid JSON that isn't an object (e.g. a bare array
     # ["ls -la"], string, or number) as function arguments. Most local tools keep
     # the legacy empty-object coercion for stream robustness, but email MCP tools
     # must fail closed so a malformed call cannot read the default mailbox.
+    # Uses the shared BUILTIN_EMAIL_TOOLS (single source of truth) so the
+    # fail-closed set can't drift from the dispatch/blocklist sets.
     if not isinstance(args, dict):
-        if tool_type.startswith("mcp__email__") or name in _BUILTIN_EMAIL_TOOLS:
+        if tool_type.startswith("mcp__email__") or name in BUILTIN_EMAIL_TOOLS:
             logger.warning(f"Non-object email function call arguments for {name}: {args!r}; rejecting")
             return None
         logger.warning(f"Non-object function call arguments for {name}: {args!r}; treating as empty")
@@ -1241,7 +1313,7 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
         content = json.dumps(args) if args else "{}"
         return ToolBlock(tool_type, content)
     # Email tools are implemented as MCP — route them to email
-    if name in _BUILTIN_EMAIL_TOOLS:
+    if name in BUILTIN_EMAIL_TOOLS:
         return ToolBlock(f"mcp__email__{name}", json.dumps(args) if args else "{}")
     if tool_type not in TOOL_TAGS:
         logger.warning(f"Unknown function call: {name}")
@@ -1373,6 +1445,9 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
             folder = args.get("folder") or value or "INBOX"
             mode = args.get("mode") or "reply"
             content = f"open_email_reply {uid} {folder} {mode}"
+            body = args.get("body") or args.get("extra") or args.get("content") or ""
+            if body:
+                content += f" {body}"
         elif action == "set_mode":
             content = f"set_mode {value or name}"
         elif action == "switch_model":

src/tool_security.py

@@ -8,10 +8,36 @@ from typing import Optional, Set
 logger = logging.getLogger(__name__)
 
 
+# Every tool exposed by the built-in email MCP server
+# (mcp_servers/email_server.py). Single source of truth: the fence tags
+# (TOOL_TAGS), bare-name dispatch (tool_execution), native-call mapping
+# (tool_schemas), and the non-admin blocklist below all derive from this set,
+# so a tool added to the email server can't become reachable under its bare
+# name without also being blocked for non-admins.
+BUILTIN_EMAIL_TOOLS = frozenset({
+    "list_email_accounts",
+    "list_emails",
+    "read_email",
+    "search_emails",
+    "send_email",
+    "reply_to_email",
+    "draft_email",
+    "draft_email_reply",
+    "ai_draft_email_reply",
+    "archive_email",
+    "delete_email",
+    "mark_email_read",
+    "bulk_email",
+    "download_attachment",
+})
+
+
 # Tools regular/public users must not execute directly. These either expose
 # server/runtime access, sensitive user data, external messaging, persistent
-# state changes, or generic loopback/integration surfaces.
-NON_ADMIN_BLOCKED_TOOLS = {
+# state changes, or generic loopback/integration surfaces. All email tools are
+# included (SECURITY.md: email/MCP capabilities are privileged admin
+# functionality).
+NON_ADMIN_BLOCKED_TOOLS = BUILTIN_EMAIL_TOOLS | {
     "bash",
     "python",
     "manage_bg_jobs",
@@ -34,10 +60,6 @@ NON_ADMIN_BLOCKED_TOOLS = {
     "manage_settings",
     "api_call",
     "app_api",
-    "send_email",
-    "reply_to_email",
-    "list_emails",
-    "read_email",
     "resolve_contact",
     "manage_contact",
     "manage_calendar",
@@ -74,8 +96,20 @@ PLAN_MODE_READONLY_TOOLS = {
     "search_chats",
     "list_models",
     "list_sessions",
+    # Read-only email tools. list_email_accounts must be here because the
+    # bare/qualified alias gate in execute_tool_block works both ways: it has
+    # a native function schema, so plan mode's schema-derived bare denylist
+    # contains it — and without this allowlist entry that bare entry would
+    # also block the qualified mcp__email__list_email_accounts call that the
+    # MCP read-only filter deliberately allows.
+    "list_email_accounts",
     "list_emails",
     "read_email",
+    # Explicitly read-only rather than allowed-by-omission: this PR makes
+    # every BUILTIN_EMAIL_TOOLS name fence-taggable, so each one must be
+    # classified — see the plan-mode partition test in
+    # tests/test_email_registry_sync.py.
+    "search_emails",
     "list_served_models",
     "list_downloads",
     "list_cached_models",
@@ -109,7 +143,14 @@ _PLAN_MODE_KNOWN_MUTATORS = {
     "manage_webhooks", "manage_tokens", "manage_settings", "manage_contact",
     "manage_calendar", "api_call", "app_api", "ui_control",
     "send_email", "reply_to_email", "bulk_email", "delete_email",
-    "archive_email", "mark_email_read", "download_model", "serve_model",
+    "archive_email", "mark_email_read",
+    # The draft tools create documents and download_attachment writes to
+    # disk — mutating. They have no native schemas (yet), so without these
+    # static entries plan-mode safety for their bare fence tags would depend
+    # entirely on the MCP read-only inventory being present and current.
+    "draft_email", "draft_email_reply", "ai_draft_email_reply",
+    "download_attachment",
+    "download_model", "serve_model",
     "stop_served_model", "cancel_download", "adopt_served_model", "serve_preset",
     "generate_image", "edit_image", "trigger_research", "manage_research",
     # Shell is never read-only-safe; block it explicitly so it stays out of plan
@@ -151,6 +192,28 @@ def plan_mode_disabled_tools() -> Set[str]:
     return (all_names | _PLAN_MODE_KNOWN_MUTATORS) - PLAN_MODE_READONLY_TOOLS
 
 
+def email_tool_policy_names(tool_name: str) -> frozenset:
+    """All policy-equivalent spellings of a tool name.
+
+    A bare built-in email tool name and its MCP-qualified mcp__email__<name>
+    form dispatch to the same email server tool, but policy sources spell
+    them either way — plan mode and the MCP settings toggle write qualified
+    names into denylists, chat-level toggles write bare ones. Every gate must
+    match against the full alias set, or a call in one spelling slips past a
+    denylist entry written in the other. Non-email names alias only to
+    themselves.
+    """
+    if not isinstance(tool_name, str):
+        return frozenset((tool_name,))
+    if tool_name in BUILTIN_EMAIL_TOOLS:
+        return frozenset((tool_name, f"mcp__email__{tool_name}"))
+    if tool_name.startswith("mcp__email__"):
+        bare = tool_name[len("mcp__email__"):]
+        if bare in BUILTIN_EMAIL_TOOLS:
+            return frozenset((tool_name, bare))
+    return frozenset((tool_name,))
+
+
 def is_public_blocked_tool(tool_name: Optional[str]) -> bool:
     """Return True when a non-admin/public user must not execute this tool.
 

src/tool_utils.py

@@ -54,6 +54,8 @@ def _parse_tool_args(content):
     if isinstance(content, str):
         try:
             args = json.loads(content) if content.strip() else {}
+            if not isinstance(args, dict):
+                args = {}
         except (json.JSONDecodeError, TypeError) as e:
             raise ValueError(str(e))
     elif isinstance(content, dict):

src/tools/calendar.py

@@ -254,6 +254,7 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
                     "calendar_href": ev.calendar_id,
                     "event_type": ev.event_type or "",
                     "importance": ev.importance or "normal",
+                    "rrule": ev.rrule or "",
                 })
             if not events:
                 response_text = f"No events between {start_dt.date().isoformat()} and {end_dt.date().isoformat()}."
@@ -268,6 +269,8 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
                         line += f" #{ev['event_type']}"
                     if ev.get("importance") and ev["importance"] != "normal":
                         line += f" !{ev['importance']}"
+                    if ev.get("rrule"):
+                        line += f" repeats({ev['rrule']})"
                     if ev.get("location"):
                         line += f" @ {ev['location']}"
                     if ev.get("calendar"):
@@ -480,6 +483,10 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
                 ev.event_type = _tag or None
             if args.get("importance") is not None:
                 ev.importance = args["importance"]
+            if args.get("rrule") is not None:
+                ev.rrule = args.get("rrule") or ""
+            elif str(args.get("repeat") or "").strip().lower() in {"none", "no", "off", "false", "single"}:
+                ev.rrule = ""
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
             if is_caldav:
                 ev.caldav_sync_pending = "update"

src/url_safety.py

@@ -79,12 +79,18 @@ def check_outbound_url(
     if not raw_ips:
         return False, "host does not resolve"
 
+    saw_ip = False
     for raw in raw_ips:
+        if not isinstance(raw, str):
+            continue
         try:
             ip = ipaddress.ip_address(raw.split("%")[0])  # strip IPv6 zone id
         except ValueError:
             continue
+        saw_ip = True
         reason = _classify(ip, block_private=block_private)
         if reason:
             return False, reason
+    if not saw_ip:
+        return False, "host does not resolve to an IP"
     return True, "ok"

static/app.js

@@ -22,7 +22,7 @@ import memoryModule from './js/memory.js';
 import voiceRecorderModule from './js/voiceRecorder.js';
 import censorModule from './js/censor.js';
 import galleryModule from './js/gallery.js';
-import tasksModule from './js/tasks.js';
+import tasksModule from './js/tasks.js?v=20260630tasksactivity';
 import calendarModule from './js/calendar.js';
 import notesModule from './js/notes.js';
 import adminModule from './js/admin.js';
@@ -39,7 +39,7 @@ import themeModule from './js/theme.js';
 // unversioned so this can't recur.
 import cookbookModule from './js/cookbook.js';
 import groupModule from './js/group.js';
-import * as researchPanelModule from './js/research/panel.js';
+import * as researchPanelModule from './js/research/panel.js?v=20260630researchthumb';
 import ttsModule from './js/tts-ai.js';
 import spinnerModule from './js/spinner.js';
 import { initKeyboardShortcuts } from './js/keyboard-shortcuts.js';
@@ -53,6 +53,71 @@ window.uiModule = uiModule;
 window.adminModule = adminModule;
 window.cookbookModule = cookbookModule;
 
+function initForegroundActivityHeartbeat() {
+  let lastSent = 0;
+  const minGapMs = 12000;
+  const send = (force = false) => {
+    if (document.visibilityState === 'hidden') return;
+    const now = Date.now();
+    if (!force && now - lastSent < minGapMs) return;
+    lastSent = now;
+    try {
+      if (navigator.sendBeacon) {
+        const body = new Blob(['{}'], { type: 'application/json' });
+        if (navigator.sendBeacon('/api/activity/heartbeat', body)) return;
+      }
+    } catch (_) {}
+    fetch('/api/activity/heartbeat', {
+      method: 'POST',
+      credentials: 'same-origin',
+      keepalive: true,
+      headers: { 'Content-Type': 'application/json' },
+      body: '{}',
+    }).catch(() => {});
+  };
+  send(true);
+  window.addEventListener('focus', () => send(true));
+  document.addEventListener('visibilitychange', () => {
+    if (document.visibilityState !== 'hidden') send(true);
+  });
+  ['pointerdown', 'keydown', 'touchstart', 'scroll'].forEach(type => {
+    window.addEventListener(type, () => send(false), { passive: true, capture: true });
+  });
+  setInterval(() => send(false), 15000);
+}
+initForegroundActivityHeartbeat();
+
+function initRailHoverLabels() {
+  const labels = {
+    'rail-search-btn': 'Search',
+    'rail-new-session': 'New',
+    'rail-delete-session': 'Delete',
+    'rail-chats': 'Chat',
+    'rail-documents': 'Docs',
+    'rail-calendar': 'Calendar',
+    'rail-compare': 'Compare',
+    'rail-cookbook': 'Cookbook',
+    'rail-research': 'Research',
+    'rail-email': 'Email',
+    'rail-gallery': 'Gallery',
+    'rail-archive': 'Library',
+    'rail-memory': 'Brain',
+    'rail-notes': 'Notes',
+    'rail-tasks': 'Tasks',
+    'rail-theme': 'Theme',
+    'rail-settings': 'Settings',
+  };
+  document.querySelectorAll('#icon-rail .icon-rail-btn').forEach(btn => {
+    if (btn.querySelector('.rail-hover-label')) return;
+    const label = labels[btn.id] || btn.getAttribute('aria-label') || btn.getAttribute('title') || '';
+    if (!label) return;
+    const span = document.createElement('span');
+    span.className = 'rail-hover-label';
+    span.textContent = String(label).replace(/\s*\([^)]*\)\s*/g, '').trim();
+    btn.appendChild(span);
+  });
+}
+
 // Redirect to login on 401 from any fetch
 const _origFetch = window.fetch;
 window.fetch = async function(...args) {
@@ -1617,6 +1682,7 @@ function initializeEventListeners() {
       // Delay tool glow-up for a staggered effect
       setTimeout(() => applyModeToToggles(mode), 500);
     }
+    window.__odysseusSetChatMode = setMode;
     agentBtn.addEventListener('click', () => {
       // Agent mode turns off research if active
       const resChk = el('research-toggle');
@@ -1692,10 +1758,29 @@ function initializeEventListeners() {
   try { workspaceModule.initWorkspace(); } catch (_) {}
 
   // Document editor toggle (special: uses module panel, not a checkbox)
+  function bringOpenDocumentToFrontOnMobile() {
+    if (window.innerWidth > 768) return false;
+    if (!documentModule || !documentModule.isPanelOpen || !documentModule.isPanelOpen()) return false;
+    if (!document.body.classList.contains('email-front')) return false;
+    document.body.classList.remove('email-front', 'email-doc-split-active');
+    document.documentElement.style.removeProperty('--email-doc-split-left-x');
+    document.documentElement.style.removeProperty('--email-doc-split-email-w');
+    document.documentElement.style.removeProperty('--email-doc-split-right-x');
+    const docPane = document.getElementById('doc-editor-pane');
+    if (docPane) docPane.style.setProperty('z-index', '10010', 'important');
+    const overflow = el('overflow-doc-btn');
+    if (overflow) overflow.classList.add('active');
+    const indicator = el('doc-indicator-btn');
+    if (indicator) indicator.classList.add('active');
+    const st = loadToggleState(); st.doc = true; saveToggleState(st);
+    return true;
+  }
+
   const overflowDocBtn = el('overflow-doc-btn');
   if (overflowDocBtn) {
     overflowDocBtn.addEventListener('click', async () => {
       if (!documentModule) return;
+      if (bringOpenDocumentToFrontOnMobile()) return;
       if (documentModule.isPanelOpen()) {
         documentModule.closePanel();
         overflowDocBtn.classList.remove('active');
@@ -2101,7 +2186,7 @@ function initializeEventListeners() {
     const pickerWrap = el('model-picker-wrap');
     if (!inputTop || !pickerWrap) return;
 
-    const PLACEHOLDER_HIDE_WIDTH = 400;
+    const PLACEHOLDER_COMPACT_WIDTH = 400;
     const PICKER_HIDE_WIDTH = 220;
     const TOOLBAR_HIDE_WIDTH = 160;
     const textarea = el('message');
@@ -2114,9 +2199,10 @@ function initializeEventListeners() {
       const w = inputTop.clientWidth;
       // Hide model picker
       pickerWrap.classList.toggle('picker-auto-hidden', w < PICKER_HIDE_WIDTH);
-      // Hide placeholder text
+      // Keep a prompt inside the composer even when the picker crowds the row.
+      // A blank placeholder makes the mobile/compact empty state feel broken.
       if (textarea) {
-        textarea.setAttribute('placeholder', w < PLACEHOLDER_HIDE_WIDTH ? '' : 'Message Odysseus...');
+        textarea.setAttribute('placeholder', w < PLACEHOLDER_COMPACT_WIDTH ? 'Message...' : 'Message Odysseus...');
       }
       // Hide entire bottom toolbar (tools, mode toggle) — only send button remains
       if (inputBottom) {
@@ -2304,14 +2390,32 @@ function initializeEventListeners() {
         // IMPORTANT: don't overwrite the user's persisted per-mode tool prefs
         // (`web_agent`, `bash_agent`, `web_chat`, `bash_chat`). Nobody mode is
         // ephemeral — their agent-mode defaults must come back on toggle-off.
+        const beforeNobody = Storage.getJSON(Storage.KEYS.TOGGLES, {}) || {};
+        if (!beforeNobody.nobody_prev_mode) beforeNobody.nobody_prev_mode = beforeNobody.mode || 'agent';
+        Storage.setJSON(Storage.KEYS.TOGGLES, beforeNobody);
         const _offIds = ['web-toggle', 'bash-toggle', 'research-toggle'];
         _offIds.forEach(id => { const c = el(id); if (c) c.checked = false; });
         ['web-toggle-btn', 'bash-toggle-btn'].forEach(id => { const b = el(id); if (b) b.classList.remove('active'); });
-        const _ab = el('mode-agent-btn'), _cb = el('mode-chat-btn');
-        if (_ab) _ab.classList.remove('active');
-        if (_cb) _cb.classList.add('active');
+        if (typeof window.__odysseusSetChatMode === 'function') {
+          window.__odysseusSetChatMode('chat');
+        } else {
+          const _ab = el('mode-agent-btn'), _cb = el('mode-chat-btn');
+          if (_ab) {
+            _ab.classList.remove('active');
+            _ab.setAttribute('aria-pressed', 'false');
+          }
+          if (_cb) {
+            _cb.classList.add('active');
+            _cb.setAttribute('aria-pressed', 'true');
+          }
+          const _toggle = _ab?.closest('.mode-toggle') || _cb?.closest('.mode-toggle');
+          if (_toggle) _toggle.classList.add('mode-chat');
+          const ts = Storage.getJSON(Storage.KEYS.TOGGLES, {});
+          ts.mode = 'chat';
+          Storage.setJSON(Storage.KEYS.TOGGLES, ts);
+        }
         const ts = Storage.getJSON(Storage.KEYS.TOGGLES, {});
-        ts.research = false; ts.mode = 'chat';
+        ts.research = false;
         Storage.setJSON(Storage.KEYS.TOGGLES, ts);
       } else {
         incognitoBtn.innerHTML = INCOGNITO_EYE_OPEN + '<span class="incognito-label">Nobody</span>';
@@ -2335,11 +2439,15 @@ function initializeEventListeners() {
         // Heal any previously-persisted false values from the old Nobody bug
         // so agent-mode defaults (web/bash ON) come back.
         const _ts = Storage.getJSON(Storage.KEYS.TOGGLES, {});
-        let _dirty = false;
+        const _restoreMode = _ts.nobody_prev_mode || 'agent';
+        delete _ts.nobody_prev_mode;
         ['web_agent', 'bash_agent', 'web_chat', 'bash_chat'].forEach(k => {
-          if (_ts[k] === false) { delete _ts[k]; _dirty = true; }
+          if (_ts[k] === false) delete _ts[k];
         });
-        if (_dirty) Storage.setJSON(Storage.KEYS.TOGGLES, _ts);
+        Storage.setJSON(Storage.KEYS.TOGGLES, _ts);
+        if (typeof window.__odysseusSetChatMode === 'function') {
+          window.__odysseusSetChatMode(_restoreMode === 'chat' ? 'chat' : 'agent');
+        }
         // Reapply the current mode's real defaults to the visible toggles
         const _curMode = (Storage.getJSON(Storage.KEYS.TOGGLES, {}) || {}).mode || 'chat';
         try { applyModeToToggles(_curMode); } catch (_) {}
@@ -3342,8 +3450,15 @@ function initializeEventListeners() {
 function startOdysseusApp() {
   if (window.__odysseusAppStarted) return;
   window.__odysseusAppStarted = true;
+  const _bumpChatPriority = (ms = 10000) => {
+    try {
+      window.__odysseusChatBusyUntil = Math.max(window.__odysseusChatBusyUntil || 0, Date.now() + ms);
+    } catch (_) {}
+  };
+  _bumpChatPriority(10000);
   // Set CSS variables
   document.documentElement.style.setProperty('--line-height', '20px');
+  initRailHoverLabels();
 
   // Smooth keyboard open/close on mobile — keep chat scrolled to bottom
   if (window.visualViewport && 'ontouchstart' in window) {
@@ -3509,9 +3624,16 @@ function startOdysseusApp() {
   const chatForm = document.getElementById('chat-form');
   const originalSubmit = chatModule.handleChatSubmit;
   let _submitting = false;
+  const _messageInput = document.getElementById('message') || document.getElementById('message-input');
+  if (_messageInput) {
+    _messageInput.addEventListener('focus', () => _bumpChatPriority(15000));
+    _messageInput.addEventListener('input', () => _bumpChatPriority(15000));
+    _messageInput.addEventListener('pointerdown', () => _bumpChatPriority(15000), { passive: true });
+  }
 
   function handleSubmit(e) {
     if (e) e.preventDefault();
+    _bumpChatPriority(30000);
     // Debounce: prevent double-submit while a request is being initiated
     if (_submitting) return;
     _submitting = true;
@@ -3928,7 +4050,8 @@ function startOdysseusApp() {
   }
 
   // Non-critical: load in parallel, resolve silently
-  modelsModule.refreshModels(true).then(() => {
+  modelsModule.refreshModels(false).then(() => {
+    try { sessionModule.updateModelPicker(); } catch (_) {}
     const modelsBox = document.getElementById('models');
     const hasModels = modelsBox && modelsBox.querySelector('.models-row');
     if (!hasModels) {

static/index.html

@@ -219,8 +219,8 @@
       }, { once: true });
     })();
   </script>
-  <link rel="stylesheet" href="/static/style.css">
-  <link rel="modulepreload" href="/static/app.js">
+  <link rel="stylesheet" href="/static/style.css?v=20260630mdfontsize">
+  <link rel="modulepreload" href="/static/app.js?v=20260630mdfontsize">
   <link rel="modulepreload" href="/static/js/chat.js">
   <link rel="modulepreload" href="/static/js/ui.js">
   <link rel="modulepreload" href="/static/js/sessions.js">
@@ -1591,8 +1591,16 @@
                settings backup. Re-add this card to surface the toggle
                again once the core experience is faster. -->
           <div class="admin-card">
-            <h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><rect x="2" y="4" width="20" height="16" rx="2"/><polyline points="2 6 12 13 22 6"/></svg>Email Safety<span style="flex:1"></span><label class="admin-switch" title="When on, agent send_email and reply_to_email tools stage a draft for your approval instead of sending immediately."><input type="checkbox" id="set-agentEmailConfirm" checked><span class="admin-slider"></span></label></h2>
-            <div class="admin-toggle-sub" style="margin-bottom:8px">When on, agent <code>send_email</code> / <code>reply_to_email</code> tools stage a draft for your approval (in the chat) instead of SMTPing immediately. Stops models from inventing a signature and sending it to a real recipient before you can review.</div>
+            <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.121 2.121 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg>Writing Style</h2>
+            <div class="admin-toggle-sub" style="margin-bottom:8px">Used when AI drafts email replies. Keep this email-specific: greetings, sign-off, tone, and length.</div>
+            <div class="settings-col">
+              <textarea id="set-email-style" rows="6" class="settings-select" style="font-family:inherit;resize:none" placeholder="e.g. I write emails in this style. I don't use exclamation marks. I sign emails with: ..."></textarea>
+              <div class="settings-row" style="margin-top:4px">
+                <span id="set-email-style-msg" style="font-size:11px;"></span>
+                <button class="admin-btn-add" id="set-email-style-extract" style="margin-left:auto;display:inline-flex;align-items:center;gap:5px;"><svg width="12" height="12" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Extract from Sent (15 emails)</button>
+                <button class="admin-btn-add" id="set-email-style-save">Save</button>
+              </div>
+            </div>
           </div>
         </div>
 
@@ -1972,17 +1980,33 @@
           </div>
 
           <div class="admin-card">
-            <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M11 4H4a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7"/><path d="M18.5 2.5a2.121 2.121 0 0 1 3 3L12 15l-4 1 1-4 9.5-9.5z"/></svg>Writing Style</h2>
-            <div class="admin-toggle-sub" style="margin-bottom:8px">AI-extracted from your sent emails. Used when AI drafts replies.</div>
-            <div class="settings-col">
-              <textarea id="set-email-style" rows="6" class="settings-select" style="font-family:inherit;resize:none" placeholder="e.g. I write emails in this style. I don't use exclamation marks. I sign emails with: ..."></textarea>
-              <div class="settings-row" style="margin-top:4px">
-                <span id="set-email-style-msg" style="font-size:11px;"></span>
-                <button class="admin-btn-add" id="set-email-style-extract" style="margin-left:auto;display:inline-flex;align-items:center;gap:5px;"><svg width="12" height="12" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>Extract from Sent (15 emails)</button>
-                <button class="admin-btn-add" id="set-email-style-save">Save</button>
+            <h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="m5 8 6 6"/><path d="m4 14 6-6 2-3"/><path d="M2 5h12"/><path d="M7 2h1"/><path d="m22 22-5-10-5 10"/><path d="M14 18h6"/></svg>Translation</h2>
+            <div class="settings-row" style="align-items:center;">
+              <div style="flex:1;min-width:0;">
+                <div class="settings-label" style="margin-bottom:3px;">Auto translate</div>
+                <div class="admin-toggle-sub" style="margin:0;">When an opened email appears to be in another language, prepare a translated view.</div>
               </div>
+              <label class="admin-switch"><input type="checkbox" id="set-email-auto-translate"><span class="admin-slider"></span></label>
+            </div>
+            <div class="settings-row" style="align-items:center;margin-top:8px;">
+              <label class="settings-label" for="set-email-translate-language">Translate to</label>
+              <input id="set-email-translate-language" class="settings-select" list="set-email-translate-language-list" placeholder="English, Swedish, Japanese..." style="max-width:220px;">
+              <datalist id="set-email-translate-language-list">
+                <option value="English"></option>
+                <option value="Swedish"></option>
+                <option value="Norwegian"></option>
+                <option value="Danish"></option>
+                <option value="Japanese"></option>
+                <option value="Spanish"></option>
+                <option value="French"></option>
+                <option value="German"></option>
+                <option value="British English"></option>
+                <option value="Plain English"></option>
+              </datalist>
+              <span id="set-email-translate-msg" style="font-size:11px;margin-left:auto;"></span>
             </div>
           </div>
+
         </div>
 
         <!-- ═══ REMINDERS TAB ═══ -->
@@ -2475,10 +2499,10 @@
 <script type="module" src="/static/js/tts-ai.js"></script>
 <script type="module" src="/static/js/document.js"></script>
 <script type="module" src="/static/js/gallery.js"></script>
-<script type="module" src="/static/js/chatRenderer.js"></script>
+<script type="module" src="/static/js/chatRenderer.js?v=20260630toolmetrics"></script>
 <script type="module" src="/static/js/codeRunner.js"></script>
 <script type="module" src="/static/js/chatStream.js"></script>
-<script type="module" src="/static/js/chat.js?v=20260609ws"></script>
+<script type="module" src="/static/js/chat.js?v=20260630toolmetrics"></script>
 <script type="module" src="/static/js/cookbook.js"></script>
 <script src="/static/js/cookbookSchedule.js"></script>
 <script type="module" src="/static/js/search-chat.js"></script>
@@ -2488,7 +2512,7 @@
 <script type="module" src="/static/js/settings.js"></script>
 <script type="module" src="/static/js/admin.js"></script>
 <script type="module" src="/static/js/assistant.js"></script>
-<script type="module" src="/static/app.js"></script>  <!-- app.js must be LAST -->
+<script type="module" src="/static/app.js?v=20260630mdfontsize"></script>  <!-- app.js must be LAST -->
 <script type="module" src="/static/js/init.js"></script>
 <script type="module" src="/static/js/a11y.js"></script>
 <script nonce="{{CSP_NONCE}}">if('serviceWorker' in navigator){navigator.serviceWorker.register('/static/sw.js').catch(()=>{});}</script>

static/js/admin.js

@@ -499,7 +499,8 @@ async function loadEndpoints() {
       return;
     }
     const rowHtml = data.map(ep => {
-      const visibleCount = ep.models.length;
+      const epModels = Array.isArray(ep.models) ? ep.models : [];
+      const visibleCount = epModels.length;
       const totalCount = visibleCount + (ep.hidden_count || 0);
       // `ep.models` is the *visible* set — when every model is hidden it's
       // empty, but we still need to render the expand panel so the user can
@@ -1393,34 +1394,77 @@ function initEndpointForm() {
     _refreshOfflineCount();
   }
 
-  const probeAllBtn = el('adm-epProbeAllBtn');
-  if (probeAllBtn) {
-    probeAllBtn.addEventListener('click', async () => {
+  const _fetchWithTimeout = async (url, opts = {}, timeoutMs = 25000) => {
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+    try {
+      return await fetch(url, { ...opts, signal: ctrl.signal });
+    } finally {
+      clearTimeout(timer);
+    }
+  };
+  const _collectAddedEndpointIds = async () => {
+    const domIds = Array.from(document.querySelectorAll('[data-adm-ep-id]'))
+      .map(r => r.getAttribute('data-adm-ep-id'))
+      .filter(Boolean);
+    if (domIds.length) return Array.from(new Set(domIds));
+    try {
+      const res = await fetch('/api/model-endpoints', { credentials: 'same-origin' });
+      const data = await res.json().catch(() => []);
+      return (Array.isArray(data) ? data : []).map(ep => ep && ep.id).filter(Boolean);
+    } catch (_) {
+      return [];
+    }
+  };
+  const _setProbeAllButtonLabel = async (btn, text, whirlpoolRef) => {
+    btn.innerHTML = '';
+    if (whirlpoolRef && whirlpoolRef.element) btn.appendChild(whirlpoolRef.element);
+    btn.appendChild(document.createTextNode(text));
+  };
+  if (!window.__admEpProbeAllWired) {
+    window.__admEpProbeAllWired = true;
+    document.addEventListener('click', async (ev) => {
+      const probeAllBtn = ev.target.closest('#adm-epProbeAllBtn');
+      if (!probeAllBtn || probeAllBtn.disabled) return;
+      ev.preventDefault();
       probeAllBtn.disabled = true;
       const origHTML = probeAllBtn.innerHTML;
       let _wp = null;
       try {
-        const sp = window.spinnerModule || (await import('./spinner.js')).default;
-        _wp = sp.createWhirlpool(11);
-        _wp.element.style.cssText = 'display:inline-flex;width:11px;height:11px;margin:0 4px 0 0;';
-        probeAllBtn.innerHTML = '';
-        probeAllBtn.appendChild(_wp.element);
-        probeAllBtn.appendChild(document.createTextNode('Probing'));
-      } catch (_) {
-        probeAllBtn.innerHTML = '<span style="opacity:0.7;">Probing…</span>';
-      }
-      try {
-        // Hit the bulk local probe (same one the model picker uses).
-        await fetch('/api/model-endpoints/probe-local', { credentials: 'same-origin' }).catch(() => {});
-        // Then per-endpoint /probe for the rest so API/cloud endpoints
-        // refresh too. Parallel — capped to 6 at a time so we don't
-        // hammer the backend on a big list.
-        const ids = Array.from(document.querySelectorAll('[data-adm-ep-id]')).map(r => r.getAttribute('data-adm-ep-id')).filter(Boolean);
+        try {
+          const sp = window.spinnerModule || (await import('./spinner.js')).default;
+          _wp = sp.createWhirlpool(11);
+          _wp.element.style.cssText = 'display:inline-flex;width:11px;height:11px;margin:0 4px 0 0;';
+          await _setProbeAllButtonLabel(probeAllBtn, 'Probing', _wp);
+        } catch (_) {
+          probeAllBtn.innerHTML = '<span style="opacity:0.7;">Probing...</span>';
+        }
+        await _fetchWithTimeout('/api/model-endpoints/probe-local', { credentials: 'same-origin' }, 12000).catch(() => null);
+        const ids = await _collectAddedEndpointIds();
+        if (!ids.length) {
+          await loadEndpoints();
+          if (uiModule && uiModule.showToast) uiModule.showToast('No endpoints to probe', 1800);
+          return;
+        }
+        let done = 0;
+        let failed = 0;
         const lane = async (id) => {
-          try { await fetch(`/api/model-endpoints/${id}/probe`, { credentials: 'same-origin' }); } catch (_) {}
+          try {
+            const res = await _fetchWithTimeout(`/api/model-endpoints/${encodeURIComponent(id)}/models?refresh=true&refresh_timeout=20`, {
+              credentials: 'same-origin'
+            }, 25000);
+            if (!res || !res.ok || res.headers.get('X-Model-Refresh-Status') === 'failed') failed += 1;
+            else await res.json().catch(() => null);
+          } catch (err) {
+            failed += 1;
+            console.warn('Endpoint probe failed', id, err);
+          } finally {
+            done += 1;
+            try { await _setProbeAllButtonLabel(probeAllBtn, `Probing ${done}/${ids.length}`, _wp); } catch (_) {}
+          }
         };
         const queue = [...ids];
-        const workers = Array.from({length: Math.min(6, queue.length)}, () => (async () => {
+        const workers = Array.from({ length: Math.min(4, queue.length) }, () => (async () => {
           while (queue.length) {
             const id = queue.shift();
             if (id) await lane(id);
@@ -1428,7 +1472,11 @@ function initEndpointForm() {
         })());
         await Promise.all(workers);
         await loadEndpoints();
-        if (uiModule && uiModule.showToast) uiModule.showToast('Endpoint status refreshed', 1800);
+        _refreshOfflineCount();
+        if (uiModule && uiModule.showToast) {
+          const ok = Math.max(0, ids.length - failed);
+          uiModule.showToast(failed ? `Probed ${ok}/${ids.length} endpoints; ${failed} failed` : `Probed ${ids.length} endpoints`, failed ? 4200 : 1800);
+        }
       } finally {
         if (_wp) { try { _wp.destroy(); } catch (_) {} }
         probeAllBtn.innerHTML = origHTML;

static/js/calendar.js

@@ -301,7 +301,7 @@ async function _updateEvent(uid, data) {
   return { ok: true };
 }
 
-async function _deleteEvent(uid) {
+async function _deleteEvent(uid, { scope = 'series' } = {}) {
   // Multiple "sibling" UIDs may need to vanish optimistically:
   //   1. The exact uid the user clicked.
   //   2. If the user clicked a RECURRING occurrence (uid contains "::"),
@@ -312,9 +312,12 @@ async function _deleteEvent(uid) {
   //      other days kept rendering until the next full refresh.
   //   3. If the user clicked the master, strip every "master::*"
   //      expansion (same prefix scan).
+  const deleteOccurrenceOnly = scope === 'occurrence' && uid.includes('::');
   const masterUid = uid.includes('::') ? uid.split('::')[0] : uid;
   const backups = {};
-  const _matches = (k) => k === uid || k === masterUid || k.startsWith(masterUid + '::');
+  const _matches = deleteOccurrenceOnly
+    ? (k) => k === uid
+    : (k) => k === uid || k === masterUid || k.startsWith(masterUid + '::');
 
   for (const k of Object.keys(_allEvents)) {
     if (_matches(k)) {
@@ -328,7 +331,8 @@ async function _deleteEvent(uid) {
   if (_open) _render();
   _updateBadge && _updateBadge();
   const isRecurring = uid.includes('::');
-  fetch(`${API_BASE}/api/calendar/events/${encodeURIComponent(uid)}`, {
+  const scopeParam = deleteOccurrenceOnly ? '?scope=occurrence' : '';
+  fetch(`${API_BASE}/api/calendar/events/${encodeURIComponent(uid)}${scopeParam}`, {
     method: 'DELETE', credentials: 'same-origin',
   }).then(r => {
     // 404 = the event was already deleted by another session/device. That's
@@ -430,14 +434,77 @@ function _todayCount() {
   }).length;
 }
 
-// Per-event ⋮ menu: Remind me / Delete
+function _findEventByUid(uid) {
+  return _allEvents[uid] || _events.find(e => e && e.uid === uid) || null;
+}
+
+function _isRecurringEvent(ev) {
+  return !!(ev && (ev.is_recurrence || ev.uid?.includes('::') || ev.rrule));
+}
+
+function _chooseRecurringDeleteScope(ev) {
+  return new Promise(resolve => {
+    const name = ev?.summary ? `"${ev.summary}"` : 'this event';
+    const overlay = document.createElement('div');
+    overlay.className = 'modal';
+    overlay.style.display = '';
+    overlay.innerHTML = `
+      <div class="modal-content styled-confirm-box" role="dialog" aria-modal="true" aria-labelledby="cal-delete-choice-title">
+        <div class="modal-header"><h4 id="cal-delete-choice-title">Delete recurring event</h4></div>
+        <div class="modal-body"><p>Delete ${_e(name)}?</p></div>
+        <div class="modal-footer" style="gap:8px;flex-wrap:wrap;">
+          <button class="confirm-btn confirm-btn-secondary" data-choice="cancel">Cancel</button>
+          <button class="confirm-btn confirm-btn-danger" data-choice="occurrence">This event only</button>
+          <button class="confirm-btn confirm-btn-danger" data-choice="series">All recurring events</button>
+        </div>
+      </div>`;
+    document.body.appendChild(overlay);
+    const close = (value) => {
+      document.removeEventListener('keydown', onKey);
+      overlay.remove();
+      resolve(value);
+    };
+    const onKey = (e) => {
+      if (e.key === 'Escape') {
+        e.preventDefault();
+        close(null);
+      }
+    };
+    overlay.addEventListener('click', (e) => {
+      if (e.target === overlay) return close(null);
+      const btn = e.target.closest('[data-choice]');
+      if (!btn) return;
+      const choice = btn.dataset.choice;
+      close(choice === 'cancel' ? null : choice);
+    });
+    document.addEventListener('keydown', onKey);
+    overlay.querySelector('[data-choice="occurrence"]')?.focus();
+  });
+}
+
+async function _confirmAndDeleteEvent(ev) {
+  if (!ev) return;
+  const name = ev.summary ? `"${ev.summary}"` : 'this event';
+  let scope = 'series';
+  if (_isRecurringEvent(ev)) {
+    scope = await _chooseRecurringDeleteScope(ev);
+    if (!scope) return;
+  } else {
+    const ok = await uiModule.styledConfirm(`Delete ${name}?`, { confirmText: 'Delete', danger: true });
+    if (!ok) return;
+  }
+  try { await _deleteEvent(ev.uid, { scope }); setTimeout(() => _render(), 100); }
+  catch (_) { uiModule.showToast('Failed to delete'); }
+}
+
+// Per-event ⋮ menu: Edit / Delete
 function _wireQuickDelete(body) {
   body.querySelectorAll('.cal-event-more').forEach(btn => {
     btn.addEventListener('click', (e) => {
       e.stopPropagation();
       const uid = btn.dataset.uid;
       if (!uid) return;
-      const ev = _allEvents[uid];
+      const ev = _findEventByUid(uid);
       if (!ev) return;
       _showEventMoreMenu(ev, btn);
     });
@@ -490,10 +557,7 @@ function _showEventMoreMenu(ev, anchor) {
 
   dropdown.appendChild(_item(_trashIcon, 'Delete', async () => {
     closeMenu();
-    const name = ev.summary ? `"${ev.summary}"` : 'this event';
-    const ok = await uiModule.styledConfirm(`Delete ${name}?`, { confirmText: 'Delete', danger: true });
-    if (!ok) return;
-    try { await _deleteEvent(ev.uid); setTimeout(() => _render(), 100); } catch (_) {}
+    await _confirmAndDeleteEvent(ev);
   }, true));
 
   document.body.appendChild(dropdown);
@@ -1814,7 +1878,7 @@ function _dayDetailHTML(dateStr) {
     </div>`;
   if (_searchQuery) {
     const q = _searchQuery.toLowerCase();
-    const results = _events
+    const results = Object.values(_allEvents || {})
       .filter(_eventVisible)
       .filter(e =>
         (e.summary || '').toLowerCase().includes(q) ||
@@ -3116,7 +3180,7 @@ function _showEventForm(existing, defaultDate, defaultEndDate) {
       all_day: isAD,
       description: document.getElementById('cal-f-desc').value,
       location: document.getElementById('cal-f-loc').value,
-      rrule: document.getElementById('cal-f-rrule').value || undefined,
+      rrule: document.getElementById('cal-f-rrule').value || '',
       calendar_href: document.getElementById('cal-f-cal')?.value || (_calendars[0]?.href || ''),
       color: colorVal || undefined,
     };
@@ -3142,11 +3206,7 @@ function _showEventForm(existing, defaultDate, defaultEndDate) {
     } catch (e) { uiModule.showToast('Failed to save'); }
   });
   document.getElementById('cal-f-del')?.addEventListener('click', async () => {
-    const name = existing && existing.summary ? `"${existing.summary}"` : 'this event';
-    const ok = await uiModule.styledConfirm(`Delete ${name}?`, { confirmText: 'Delete', danger: true });
-    if (!ok) return;
-    try { await _deleteEvent(existing.uid); _render(); }
-    catch (e) { uiModule.showToast('Failed to delete'); }
+    await _confirmAndDeleteEvent(existing);
   });
   // ── Bespoke-form behavior ──────────────────────────────────────────
   const formEl = body.querySelector('.cal-form');
@@ -3455,8 +3515,18 @@ function openCalendar() {
       // Layer Esc: close the topmost calendar surface first, only fall through
       // to closing the whole calendar when nothing else is on top.
       const settings = document.getElementById('cal-settings-panel');
-      if (settings) { settings.remove(); return; }
-      if (document.querySelector('.cal-form')) { _render(); return; }
+      if (settings) {
+        e.preventDefault();
+        e.stopPropagation();
+        settings.remove();
+        return;
+      }
+      if (document.querySelector('.cal-form')) {
+        e.preventDefault();
+        e.stopPropagation();
+        _render();
+        return;
+      }
       closeCalendar();
     }
     else if (e.key === 'ArrowLeft') document.getElementById('cal-prev')?.click();
@@ -3485,14 +3555,25 @@ async function openCalendarTo(target) {
   if (!target) return;
   try {
     await _fetchCalendars();
+    const targetStr = String(target || '').trim();
+    if (targetStr.startsWith('search:')) {
+      _searchQuery = targetStr.slice('search:'.length).trim();
+      const now = new Date();
+      await _fetchEvents(`${now.getFullYear()}-01-01`, `${now.getFullYear() + 2}-01-01`);
+      _currentDate = now;
+      _selectedDay = new Date(now.getFullYear(), now.getMonth(), now.getDate());
+      _view = 'month';
+      _render();
+      return;
+    }
     // If target looks like an ISO date (YYYY-MM-DD...), go straight there.
     let dt = null;
-    const isoMatch = /^\d{4}-\d{2}-\d{2}/.test(String(target));
+    const isoMatch = /^\d{4}-\d{2}-\d{2}/.test(targetStr);
     if (isoMatch) {
-      dt = new Date(target);
+      dt = new Date(targetStr);
     } else {
       // Treat as an event uid — find it among loaded events.
-      const ev = (_events || []).find(e => e.uid === target || (e.uid || '').startsWith(target));
+      const ev = Object.values(_allEvents || {}).find(e => e.uid === targetStr || (e.uid || '').startsWith(targetStr));
       if (ev && ev.dtstart) dt = new Date(ev.dtstart);
       if (ev) _highlightEventUid = ev.uid;
     }

static/js/chat.js

@@ -43,6 +43,13 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
   let _sendInFlight = false;   // covers the window from click → streaming start
   let _displayOverride = null; // Override visible user bubble text (hides injected prompts)
   let _hideUserBubble = false; // Skip user bubble entirely (e.g. continue after stop)
+
+  function _setForegroundChatBusy(active) {
+    try {
+      window.__odysseusChatBusy = !!active;
+      window.__odysseusChatBusyUntil = active ? Date.now() + 120000 : Date.now() + 1200;
+    } catch (_) {}
+  }
   let _pendingContinue = null; // Stores the stopped AI element to merge with new response
   // ── Auto-recovery: when a turn's stream silently dies (connection drop) or
   // goes quiet while the connection is alive, re-engage the model with a
@@ -99,6 +106,94 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     const body = msgEl.querySelector('.body');
     if (body) chatRenderer.appendReportButton(body, sessionId);
   }
+
+  function _stripDocumentFenceForChat(text, { final = false } = {}) {
+    let s = String(text || '').replace(/<?\|end\|>?/g, '');
+    const markerMatch = /```(?:create_document|documen(?:t)?)\s*\n/i.exec(s);
+    if (!markerMatch) return s;
+    const before = s.slice(0, markerMatch.index).trimEnd();
+    const fenceStart = markerMatch.index;
+    const openingEnd = s.indexOf('\n', fenceStart);
+    const closeIdx = openingEnd >= 0 ? s.indexOf('\n```', openingEnd + 1) : -1;
+    const after = closeIdx >= 0 ? s.slice(closeIdx + 4).trimStart() : '';
+    const visible = [before, after].filter(Boolean).join('\n\n').trim();
+    return final && !visible ? 'Done.' : visible;
+  }
+
+  function _showDocumentWritingStatus(contentEl) {
+    const msg = contentEl && contentEl.closest ? contentEl.closest('.msg') : null;
+    const chatBox = document.getElementById('chat-history');
+    if (!msg || !chatBox) {
+      if (contentEl) contentEl.textContent = 'Writing...';
+      return;
+    }
+    let thread = msg._docWritingThread;
+    if (!thread || !thread.isConnected) {
+      thread = document.createElement('div');
+      thread.className = 'agent-thread streaming has-bottom';
+      thread.dataset.docWriting = '1';
+      const prev = msg.previousElementSibling;
+      if (prev && (prev.classList.contains('msg') || prev.classList.contains('agent-thread'))) {
+        thread.classList.add('has-top');
+      }
+      const node = document.createElement('div');
+      node.className = 'agent-thread-node running';
+      node.innerHTML = '<div class="agent-thread-dot"></div><div class="agent-thread-header"><span class="agent-thread-icon">▶</span><span class="agent-thread-tool">Writing</span><span class="agent-thread-wave">▁▂▃</span></div><div class="agent-thread-content"></div>';
+      thread.appendChild(node);
+      chatBox.insertBefore(thread, msg);
+      msg._docWritingThread = thread;
+
+      const waveEl = node.querySelector('.agent-thread-wave');
+      if (waveEl) {
+        const waveFrames = ['▁▂▃', '▂▃▄', '▃▄▅', '▄▅▆', '▅▆▇', '▆▅▄', '▅▄▃', '▄▃▂'];
+        let waveIdx = 0;
+        node._waveInterval = setInterval(() => {
+          waveIdx = (waveIdx + 1) % waveFrames.length;
+          waveEl.textContent = waveFrames[waveIdx];
+        }, 100);
+      }
+      node._startTime = Date.now();
+      node._elapsedTicker = setInterval(() => {
+        const hdr = node.querySelector('.agent-thread-header');
+        if (!hdr) return;
+        let el = hdr.querySelector('.agent-thread-elapsed');
+        if (!el) {
+          el = document.createElement('span');
+          el.className = 'agent-thread-elapsed';
+          const icon = hdr.querySelector('.agent-thread-icon');
+          if (icon && icon.nextSibling) hdr.insertBefore(el, icon.nextSibling);
+          else hdr.appendChild(el);
+        }
+        const s = (Date.now() - node._startTime) / 1000;
+        el.textContent = s < 60 ? `${s.toFixed(2)}s` : `${Math.floor(s / 60)}m ${(s % 60).toFixed(2).padStart(5, '0')}s`;
+      }, 50);
+    }
+    msg.style.display = 'none';
+  }
+
+  function _finishDocumentWritingStatus(msg, ok = true) {
+    const thread = msg && msg._docWritingThread;
+    if (!thread || !thread.isConnected) return;
+    thread.classList.remove('streaming');
+    const node = thread.querySelector('.agent-thread-node');
+    if (!node) return;
+    if (node._waveInterval) { clearInterval(node._waveInterval); node._waveInterval = null; }
+    if (node._elapsedTicker) { clearInterval(node._elapsedTicker); node._elapsedTicker = null; }
+    node.classList.remove('running');
+    if (!ok) node.classList.add('error');
+    const icon = node.querySelector('.agent-thread-icon');
+    if (icon) icon.textContent = ok ? '✓' : '✗';
+    const wave = node.querySelector('.agent-thread-wave');
+    if (wave) wave.remove();
+    if (!node.querySelector('.agent-thread-status')) {
+      const status = document.createElement('span');
+      status.className = 'agent-thread-status';
+      status.textContent = ok ? 'done' : 'failed';
+      const header = node.querySelector('.agent-thread-header');
+      if (header) header.appendChild(status);
+    }
+  }
+
   let currentAccumulated = ''; // Track accumulated text across function scope
   let currentHolder = null; // Track current message holder
   let currentSpinner = null; // Track current spinner for stop cleanup
@@ -241,12 +336,14 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       submitBtn.dataset.mode = 'streaming';
       submitBtn.dataset.phase = 'processing';
       isStreaming = true;
+      _setForegroundChatBusy(true);
       _startStallWatchdog();
     } else if (state === 'idle') {
       submitBtn.dataset.mode = '';
       delete submitBtn.dataset.phase;
       submitBtn.classList.remove('recording');
       isStreaming = false;
+      _setForegroundChatBusy(false);
       _stopStallWatchdog();
       // Defer to global updater which handles mic/newchat/send modes
       if (window._updateSendBtnIcon) {
@@ -267,6 +364,126 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
   // API key pattern for the guard in handleChatSubmit
   const API_KEY_RE = /^(sk-[a-zA-Z0-9_\-]{20,}|gsk_[a-zA-Z0-9]{20,}|AIza[a-zA-Z0-9_\-]{30,}|xai-[a-zA-Z0-9]{20,})$/;
 
+  const _queuedAgentRequests = [];
+  let _queuedDrainTimer = null;
+  let _queuedPromoteTimer = null;
+  let _queuedRequestSeq = 0;
+  let _queuedBubbleHost = null;
+
+  function _escapeQueueText(s) {
+    return String(s || '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+  }
+
+  function _ensureQueuedBubbleHost() {
+    const chatBox = document.getElementById('chat-history');
+    if (!chatBox) return null;
+    if (_queuedBubbleHost && _queuedBubbleHost.isConnected) return _queuedBubbleHost;
+    let host = document.getElementById('chat-queued-bubble-host');
+    if (!host) {
+      host = document.createElement('div');
+      host.id = 'chat-queued-bubble-host';
+      host.className = 'chat-queued-bubble-host';
+    }
+    chatBox.appendChild(host);
+    _queuedBubbleHost = host;
+    return host;
+  }
+
+  function _createQueuedBubble(item) {
+    const host = _ensureQueuedBubbleHost();
+    if (!host) return null;
+    const wrap = document.createElement('div');
+    wrap.className = 'msg msg-user msg-user-queued';
+    wrap.dataset.queueId = item.id;
+    wrap.title = 'Queued - click to send now and stop the current response';
+    wrap.innerHTML = `<div class="role">You <span class="queued-pill"><svg width="8" height="8" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><polygon points="6 4 20 12 6 20 6 4"></polygon></svg>Queued</span></div><div class="body">${_escapeQueueText(item.message)}</div>`;
+    wrap.addEventListener('click', (ev) => {
+      if (ev.target && ev.target.closest && ev.target.closest('button, a, textarea, input')) return;
+      _promoteQueuedRequest(item.id);
+    });
+    host.appendChild(wrap);
+    uiModule.scrollHistory();
+    return wrap;
+  }
+
+  function _removeQueuedRequest(id) {
+    const idx = _queuedAgentRequests.findIndex(item => item.id === id);
+    if (idx < 0) return null;
+    const [item] = _queuedAgentRequests.splice(idx, 1);
+    if (item && item.el && item.el.parentNode) item.el.remove();
+    return item;
+  }
+
+  function _setComposerAndSend(message) {
+    const input = uiModule.el('message');
+    if (!input) return false;
+    input.value = message;
+    input.dispatchEvent(new Event('input', { bubbles: true }));
+    if (uiModule.autoResize) uiModule.autoResize(input);
+    setTimeout(() => {
+      handleChatSubmit({ preventDefault() {} }).catch(err => {
+        console.error('queued send failed', err);
+        try { uiModule.showError && uiModule.showError('Queued send failed: ' + (err?.message || err)); } catch (_) {}
+      });
+    }, 0);
+    return true;
+  }
+
+  function _sendQueuedWhenIdle(item) {
+    if (!item) return;
+    const trySend = () => {
+      if (isStreaming || _sendInFlight) {
+        _queuedPromoteTimer = setTimeout(trySend, 220);
+        return;
+      }
+      _queuedPromoteTimer = null;
+      _setComposerAndSend(item.message);
+    };
+    if (_queuedPromoteTimer) clearTimeout(_queuedPromoteTimer);
+    _queuedPromoteTimer = setTimeout(trySend, 320);
+  }
+
+  function _promoteQueuedRequest(id) {
+    const item = _removeQueuedRequest(id);
+    if (!item) return;
+    if (!isStreaming && !_sendInFlight) {
+      _setComposerAndSend(item.message);
+      return;
+    }
+    try { uiModule.showToast && uiModule.showToast('Sending queued request now'); } catch (_) {}
+    const input = uiModule.el('message');
+    const submitBtn = document.querySelector('.send-btn');
+    if (input) {
+      input.value = '';
+      input.dispatchEvent(new Event('input', { bubbles: true }));
+    }
+    if (submitBtn) submitBtn.click();
+    _sendQueuedWhenIdle(item);
+  }
+
+  function _queueAgentRequest(message) {
+    const msg = String(message || '').trim();
+    if (!msg) return false;
+    const item = { id: `q${++_queuedRequestSeq}`, message: msg, createdAt: Date.now(), el: null };
+    item.el = _createQueuedBubble(item);
+    _queuedAgentRequests.push(item);
+    try { uiModule.showToast && uiModule.showToast(_queuedAgentRequests.length === 1 ? 'Queued for after this response' : `${_queuedAgentRequests.length} requests queued`); } catch (_) {}
+    return true;
+  }
+
+  function _drainQueuedAgentRequests() {
+    if (isStreaming || _sendInFlight || !_queuedAgentRequests.length) return;
+    if (_queuedDrainTimer) return;
+    _queuedDrainTimer = setTimeout(() => {
+      _queuedDrainTimer = null;
+      if (isStreaming || _sendInFlight || !_queuedAgentRequests.length) return;
+      const next = _queuedAgentRequests[0];
+      if (!next) return;
+      _removeQueuedRequest(next.id);
+      _setComposerAndSend(next.message);
+    }, 180);
+  }
+
 
   /**
    * Handle chat form submission
@@ -290,8 +507,26 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       return;
     }
 
-    // If currently streaming, stop it
+    // If currently streaming, a non-empty composer means "queue this next".
+    // Empty composer keeps the existing Stop behavior.
     if (isStreaming) {
+      const queuedInput = uiModule.el('message');
+      const queuedText = (queuedInput && queuedInput.value || '').trim();
+      if (queuedText) {
+        if (fileHandlerModule.getPendingCount && fileHandlerModule.getPendingCount()) {
+          try { uiModule.showError && uiModule.showError('Finish the current response before queueing messages with attachments.'); } catch (_) {}
+          return;
+        }
+        if (_queueAgentRequest(queuedText)) {
+          queuedInput.value = '';
+          queuedInput.dispatchEvent(new Event('input', { bubbles: true }));
+          if (uiModule.autoResize) uiModule.autoResize(queuedInput);
+        }
+        return;
+      }
+      if (fileHandlerModule.isUploading && fileHandlerModule.isUploading()) {
+        fileHandlerModule.cancelUpload && fileHandlerModule.cancelUpload();
+      }
       // Cancel server-side research if in progress
       const _cancelSid = sessionModule.getCurrentSessionId();
       if (_cancelSid && _researchingStreamIds.has(_cancelSid)) {
@@ -340,6 +575,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         const messageInput = uiModule.el('message');
         if (messageInput) messageInput.disabled = false;
         currentAccumulated = '';
+        _drainQueuedAgentRequests();
         return;
       }
       // Render whatever was accumulated so far
@@ -417,6 +653,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     // --- Send-path entry: block re-clicks between submit and stream start ---
     if (_sendInFlight) return;
     _sendInFlight = true;
+    _setForegroundChatBusy(true);
     // Instant visual feedback so the user sees their click was accepted
     // even before the streaming button state kicks in below.
     const _earlyMessageInput = uiModule.el('message');
@@ -424,6 +661,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     if (submitBtn) submitBtn.classList.add('send-pending');
     const _releaseSendFlag = () => {
       _sendInFlight = false;
+      _setForegroundChatBusy(isStreaming);
       if (_earlyMessageInput) _earlyMessageInput.disabled = false;
       if (submitBtn) submitBtn.classList.remove('send-pending');
     };
@@ -683,10 +921,19 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
       let ids = [];
       try {
-        ids = await fileHandlerModule.uploadPending();
+        ids = await fileHandlerModule.uploadPending({ sessionId: sessionModule.getCurrentSessionId() });
       } catch(e) {
         console.error('upload failed', e);
       }
+      if (_pendingAttachInfo && !ids.length && !(_pendingRegenAttachments && _pendingRegenAttachments.length)) {
+        if (_userMsgEl && _userMsgEl.parentNode) _userMsgEl.remove();
+        if (fileHandlerModule.wasLastUploadCancelled && !fileHandlerModule.wasLastUploadCancelled()) {
+          uiModule.showError && uiModule.showError('Upload failed. Attachment kept so you can retry.');
+        }
+        updateSubmitButton('idle', submitBtn);
+        _releaseSendFlag();
+        return;
+      }
 
       // Carry over the original message's file-ids on a regenerate so the new
       // send still references the same photos / docs (and picks up the user's
@@ -768,7 +1015,16 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       }
 
       // Auto-save document editor content before sending so the AI sees latest text
-      if (documentModule && documentModule.isPanelOpen() && documentModule.getCurrentDocId()) {
+      const activeEmailComposerCtx = documentModule && typeof documentModule.getActiveEmailComposerContext === 'function'
+        ? documentModule.getActiveEmailComposerContext()
+        : null;
+      let activeDocIdForSend = documentModule && typeof documentModule.getCurrentDocId === 'function'
+        ? documentModule.getCurrentDocId()
+        : null;
+      if (!activeDocIdForSend && activeEmailComposerCtx?.docId) {
+        activeDocIdForSend = activeEmailComposerCtx.docId;
+      }
+      if (documentModule && activeDocIdForSend) {
         try { await documentModule.saveDocument(); } catch(e) { console.warn('doc auto-save failed', e); }
       }
 
@@ -800,9 +1056,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       fd.append('session', streamSessionId);
       if (ids.length) fd.append('attachments', JSON.stringify(ids));
       // Auto-save & send active doc ID so the backend sees latest content
-      if (documentModule && documentModule.isPanelOpen() && documentModule.getCurrentDocId()) {
+      if (documentModule && activeDocIdForSend) {
         try { await documentModule.saveDocument({ silent: true }); } catch (_e) { /* best-effort */ }
-        fd.append('active_doc_id', documentModule.getCurrentDocId());
+        fd.append('active_doc_id', activeDocIdForSend);
       }
       // Active email context — when an email reader is open, pass its
       // uid/folder/account so "reply", "summarize", "what does this say"
@@ -811,7 +1067,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       try {
         const getEmailCtx = window.__odysseusGetActiveEmailContext;
         const emCtx = typeof getEmailCtx === 'function' ? getEmailCtx() : null;
-        if (emCtx && emCtx.uid) {
+        if (activeEmailComposerCtx && activeEmailComposerCtx.sourceUid) {
+          fd.append('active_email_uid', String(activeEmailComposerCtx.sourceUid));
+          fd.append('active_email_folder', String(activeEmailComposerCtx.sourceFolder || 'INBOX'));
+        } else if (emCtx && emCtx.uid) {
           fd.append('active_email_uid', String(emCtx.uid));
           fd.append('active_email_folder', String(emCtx.folder || 'INBOX'));
           if (emCtx.account) fd.append('active_email_account', String(emCtx.account));
@@ -820,9 +1079,11 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       // Web toggle: pre-search in Chat mode, tool permission in Agent mode
       const toggleState = Storage.loadToggleState();
       let isAgentMode = (toggleState.mode || 'chat') === 'agent';
+      const incognitoChk = el('incognito-toggle');
+      const isIncognito = !!(incognitoChk && incognitoChk.checked);
       // Auto-escalate to agent mode when a document is open — the user expects
       // the AI to see the document and have tools to edit it
-      if (!isAgentMode && documentModule && documentModule.isPanelOpen() && documentModule.getCurrentDocId()) {
+      if (!isIncognito && !isAgentMode && documentModule && activeDocIdForSend) {
         isAgentMode = true;
       }
       fd.append('mode', isAgentMode ? 'agent' : 'chat');
@@ -845,8 +1106,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       if (ragChk && !ragChk.checked) {
         fd.append('use_rag', 'false');
       }
-      const incognitoChk = el('incognito-toggle');
-      if (incognitoChk && incognitoChk.checked) {
+      if (isIncognito) {
         fd.append('incognito', 'true');
       }
       const _ws = (Storage.KEYS && Storage.get(Storage.KEYS.WORKSPACE, '')) || '';
@@ -1032,6 +1292,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       let roundHolder = holder;       // Current AI text bubble (changes per round)
       let roundText = '';             // Text accumulated for current round
       let currentToolBubble = null;   // Current tool execution bubble
+      let lastToolThread = null;      // Visible tool timeline for tool-only turns
       let roundFinalized = false;     // Whether current round's text is finalized
       let _sourcesHtml = '';          // Sources box HTML to prepend to body
       let _sourcesExpanded = false;   // Track if user expanded sources during stream
@@ -1039,6 +1300,14 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       let _sourcesType = '';          // 'web' or 'research'
       let _findingsData = null;      // Raw findings data for collapsible box
       // _keepResearchOn removed — clarification state now persisted server-side via DB mode
+      function _metricsTargetForTurn() {
+        const visibleRound = (roundHolder && roundHolder.style.display !== 'none') ? roundHolder : null;
+        const visibleText = visibleRound ? (visibleRound.querySelector('.body')?.textContent || '').trim() : '';
+        if (lastToolThread && lastToolThread.isConnected && (!visibleRound || !visibleText || visibleText === 'Done.')) {
+          return lastToolThread;
+        }
+        return visibleRound || holder;
+      }
       // Insert sources box as a stable DOM node that won't be replaced during streaming.
       // Returns the content container to use for innerHTML updates.
       function _ensureStreamLayout(body) {
@@ -1070,12 +1339,14 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         'web_search': 'Searching',
         'bash': 'Running',
         'python': 'Running',
-        'create_document': 'Writing',
-        'update_document': 'Writing',
         'read_document': 'Reading',
         'edit_file': 'Editing',
         'read_file': 'Reading',
         'write_file': 'Writing',
+        'create_document': 'Writing',
+        'edit_document': 'Editing',
+        'update_document': 'Rewriting',
+        'suggest_document': 'Reviewing',
         'list_files': 'Browsing',
         'image_gen': 'Generating',
         'generate_image': 'Generating',
@@ -1174,7 +1445,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
       // Direct render helper for streaming text
       _renderStream = () => {
-        let dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(roundText));
+        let dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(_stripDocumentFenceForChat(roundText)));
         const bodyEl = roundHolder.querySelector('.body');
         const contentEl = _ensureStreamLayout(bodyEl);
 
@@ -1253,6 +1524,11 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         // what keeps code-block hover buttons from flickering and avoids the O(N^2)
         // re-parse/re-highlight of the whole message on every token.
         // See streamingRenderer.js / streamingSegmenter.js.
+        if (_docFenceOpened && !dt.trim()) {
+          _showDocumentWritingStatus(contentEl);
+          uiModule.scrollHistory();
+          return;
+        }
         const renderer = contentEl._streamRenderer ||
           (contentEl._streamRenderer = createStreamRenderer(contentEl, {
             render: (t) => markdownModule.processWithThinking(markdownModule.squashOutsideCode(t)),
@@ -1316,7 +1592,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
               _streamSawDone = true;
               // Always update background map if entry exists (even if user switched back)
               var bgDone = _backgroundStreams.get(streamSessionId);
-              if (bgDone) {
+              if (bgDone && !_isBg) {
+                _backgroundStreams.delete(streamSessionId);
+              } else if (bgDone) {
                 bgDone.status = 'completed';
                 bgDone.accumulated = accumulated;
                 if (_isBg) {
@@ -1440,9 +1718,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 }
 
                 // --- Text-fence doc streaming (for models that don't use native tool calls) ---
-                if (!_docFenceOpened && documentModule && roundText.includes('```create_document\n')) {
-                  const fenceIdx = roundText.indexOf('```create_document\n');
-                  const afterFence = roundText.slice(fenceIdx + '```create_document\n'.length);
+                if (!_docFenceOpened && documentModule && (roundText.includes('```create_document\n') || roundText.includes('```document\n') || roundText.includes('```documen\n'))) {
+                  const fenceMarker = roundText.includes('```document\n') ? '```document\n' : (roundText.includes('```documen\n') ? '```documen\n' : '```create_document\n');
+                  const fenceIdx = roundText.indexOf(fenceMarker);
+                  const afterFence = roundText.slice(fenceIdx + fenceMarker.length);
                   const fenceLines = afterFence.split('\n');
                   if (fenceLines.length >= 1 && fenceLines[0].trim()) {
                     _docFenceOpened = true;
@@ -1451,7 +1730,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                     const knownLangs = ['python','py','javascript','js','typescript','ts','html','css','json','yaml','bash','sql','rust','go','java','c','cpp','markdown','text','plain','ruby','swift','kotlin','php','email','csv','xml','toml','ini'];
                     const isLang = fenceLines.length >= 2 && knownLangs.includes(fenceLines[1].trim().toLowerCase());
                     const lang = isLang ? fenceLines[1].trim() : '';
-                    _docFenceContentStart = fenceIdx + '```create_document\n'.length + title.length + 1 + (isLang ? fenceLines[1].length + 1 : 0);
+                    _docFenceContentStart = fenceIdx + fenceMarker.length + title.length + 1 + (isLang ? fenceLines[1].length + 1 : 0);
                     documentModule.streamDocOpen(title, lang);
                   }
                 }
@@ -1538,9 +1817,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                         <div class="thinking-header-left"><span class="live-think-header-text">Thinking\u2026</span></div>
                         <span class="live-think-spinner-slot" style="flex-shrink:0;margin-left:auto;"></span>
                         <span class="live-think-timer" style="font-size:11px;opacity:0.4;font-variant-numeric:tabular-nums;margin-left:6px;margin-right:5px;"></span>
-                        <span class="thinking-toggle live-think-toggle" id="${_liveThinkDomId}-toggle"></span>
+                        <span class="thinking-toggle live-think-toggle expanded" id="${_liveThinkDomId}-toggle"></span>
                       </div>
-                      <div class="thinking-content" id="${_liveThinkDomId}">
+                      <div class="thinking-content expanded" id="${_liveThinkDomId}">
                         <div class="thinking-content-inner live-think-inner"></div>
                       </div>
                     </div>`;
@@ -1586,13 +1865,15 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                       _liveThinkTimerEl.textContent = _formatThinkStats(_elapsedLive, _liveThinkTokenCount);
                     }
                     // Keep thinking box scrolled to bottom, but let user scroll up
+                    var _followThinking = true;
                     var thinkBox = _liveThinkInner.closest('.thinking-content');
                     if (thinkBox) {
                       var nearBottom = thinkBox.scrollHeight - thinkBox.clientHeight - thinkBox.scrollTop < 80;
                       if (nearBottom) thinkBox.scrollTop = thinkBox.scrollHeight;
+                      _followThinking = nearBottom;
                     }
                   }
-                  uiModule.scrollHistory();
+                  if (_followThinking) uiModule.scrollHistory();
                   continue;
                 } else if (!hasUnclosedThink && isThinking) {
                   isThinking = false;
@@ -2015,6 +2296,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                   if (bgM) bgM.metrics = json.data;
                   continue;
                 }
+                if (metrics) {
+                  const metricsTarget = _metricsTargetForTurn();
+                  if (metricsTarget) displayMetrics(metricsTarget, metrics);
+                }
 
               } else if (json.type === 'message_saved') {
                 // Wire the persisted DB id onto the just-streamed bubble so it
@@ -2046,7 +2331,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                 if (!roundFinalized) {
                   roundFinalized = true;
                   if (spinner && spinner.element) spinner.destroy();
-                  const dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(roundText));
+                  const dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(_stripDocumentFenceForChat(roundText)));
                   if (dt.trim()) {
                     var _body3 = roundHolder.querySelector('.body');
                     var _contentEl3 = _ensureStreamLayout(_body3);
@@ -2095,6 +2380,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
                   chatBox.appendChild(threadWrap);
                 }
                 threadWrap.classList.add('streaming');
+                lastToolThread = threadWrap;
                 const toolLabel = _toolLabels[json.tool.toLowerCase()] || json.tool;
                 const toolIcon = _toolIcons[json.tool.toLowerCase()] || '\u25B6';
                 const node = document.createElement('div')
@@ -2453,6 +2739,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
       }
 
       _renderStream();
+      if (spinner && spinner.element) { try { spinner.destroy(); } catch (_) {} spinner = null; }
       _cancelThinkingTimer();
       _removeThinkingSpinner();
       // Stop any thread pulse animations
@@ -2514,9 +2801,13 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
         // Clear streaming minHeight lock
         const _streamContent = roundHolder.querySelector('.stream-content');
         if (_streamContent) _streamContent.style.minHeight = '';
+        if (_docFenceOpened) {
+          _finishDocumentWritingStatus(roundHolder, true);
+          roundHolder.style.display = '';
+        }
 
         // Finalize the last round's bubble — flatten stream-content wrapper for clean DOM
-        const finalDisplay = stripToolBlocks(roundText);
+        const finalDisplay = stripToolBlocks(_stripDocumentFenceForChat(roundText, { final: _docFenceOpened }));
         if (finalDisplay.trim()) {
           var _body4 = roundHolder.querySelector('.body');
           // Preserve sources expanded state before final render
@@ -2632,7 +2923,9 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
         // Attach footer to the last visible bubble (roundHolder for multi-round agent, holder for single)
         const footerTarget = (roundHolder && roundHolder !== holder && roundHolder.style.display !== 'none') ? roundHolder : holder;
-        footerTarget.appendChild(createMsgFooter(footerTarget));
+        if (!footerTarget.querySelector('.msg-footer')) {
+          footerTarget.appendChild(createMsgFooter(footerTarget));
+        }
         // Add "View Report" link for completed research
         if (_researchingStreamIds.has(streamSessionId)) {
           _appendViewReportLink(footerTarget, streamSessionId);
@@ -2672,7 +2965,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
           }
         }
         if (metrics) {
-          displayMetrics(footerTarget, metrics);
+          displayMetrics(_metricsTargetForTurn() || footerTarget, metrics);
         }
         // Attach variant navigation if this was a regeneration
         _attachVariantNav(footerTarget);
@@ -2986,6 +3279,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
           sessionModule.loadSessions();
         }
       }, 3000);
+      _drainQueuedAgentRequests();
     }
   }
 
@@ -3193,6 +3487,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     // Clear local state WITHOUT aborting the fetch
     currentAbort = null;
     isStreaming = false;
+    _setForegroundChatBusy(false);
     currentHolder = null;
     currentAccumulated = '';
     // Reset submit button so the new chat is ready to send
@@ -3255,6 +3550,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     const decoder = new TextDecoder();
     let buffer = '';
     let roundText = '';
+    let docFenceOpened = false;
     let gotDelta = false;
     let leftSession = false;
     let metricsData = null;
@@ -3269,8 +3565,12 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     };
 
     const renderDelta = () => {
-      const dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(roundText));
-      contentDiv.innerHTML = markdownModule.mdToHtml(markdownModule.squashOutsideCode(dt));
+      const dt = markdownModule.normalizeThinkingMarkup(stripToolBlocks(_stripDocumentFenceForChat(roundText, { final: docFenceOpened })));
+      if (docFenceOpened && !dt.trim()) {
+        _showDocumentWritingStatus(contentDiv);
+      } else {
+        contentDiv.innerHTML = markdownModule.mdToHtml(markdownModule.squashOutsideCode(dt));
+      }
       uiModule.scrollHistory();
     };
 
@@ -3301,6 +3601,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
           try { json = JSON.parse(payload); } catch (_) { continue; }
           if (json.delta) {
             roundText += json.delta;
+            if (!docFenceOpened && (roundText.includes('```create_document\n') || roundText.includes('```document\n') || roundText.includes('```documen\n'))) {
+              docFenceOpened = true;
+              rich = true;
+            }
             if (!gotDelta) { gotDelta = true; try { spinner.destroy(); } catch (_) {} }
             renderDelta();
           } else if (json.type === 'doc_stream_open') {
@@ -3308,7 +3612,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
             if (documentModule) documentModule.streamDocOpen(json.title || '', json.lang || '');
           } else if (json.type === 'doc_stream_delta') {
             rich = true;
-            if (documentModule && json.delta) documentModule.streamDocDelta(json.delta);
+            if (documentModule) documentModule.streamDocDelta(json.content || json.delta || '');
           } else if (json.type === 'metrics') {
             metricsData = json.data || metricsData;
           } else if (json.type === 'tool_start' || json.type === 'tool_output' ||
@@ -3325,6 +3629,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     }
 
     cleanup();
+    if (docFenceOpened) _finishDocumentWritingStatus(holder, true);
     if (leftSession) { if (holder.parentNode) holder.remove(); return true; }
 
     const onThisSession = sessionModule.getCurrentSessionId &&
@@ -3344,6 +3649,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
 
     // Rich response (tools, sources, docs, multi-round) or user moved on:
     // reload from the DB for the full canonical render.
+    if (holder._docWritingThread && holder._docWritingThread.parentNode) holder._docWritingThread.remove();
     if (holder.parentNode) holder.remove();
     if (onThisSession) sessionModule.selectSession(sessionId);
     else sessionModule.loadSessions();
@@ -4137,7 +4443,10 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer
     if (!sessionId) return;
     try {
       const res = await fetch(`${API_BASE}/api/research/status/${sessionId}`);
-      if (!res.ok) return; // 404 = no research for this session
+      if (!res.ok) {
+        if (sessionModule && sessionModule.clearResearching) sessionModule.clearResearching(sessionId);
+        return; // 404 = no research for this session
+      }
       const data = await res.json();
 
       if (data.status === 'done') {

static/js/chatRenderer.js

@@ -81,7 +81,7 @@ function _formatSize(bytes) {
 // Build the `.attach-cards` element for a message's attachment list. Shared by
 // addMessage and updateMessageAttachments so a live (optimistic) user bubble
 // can be re-rendered with real upload ids once the upload resolves.
-function buildAttachCards(attachments) {
+export function buildAttachCards(attachments) {
   const attachWrap = document.createElement('div');
   attachWrap.className = 'attach-cards';
   for (const att of attachments) {
@@ -425,6 +425,23 @@ const TOOL_CALL_RE = /\[TOOL_CALL\][\s\S]*?\[\/TOOL_CALL\]/gi;
 let EXEC_FENCE_RE = null;
 const EXEC_FENCE_NON_TOOL = new Set(['bash', 'python']);
 
+function escapeRegex(source) {
+  return String(source).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function stripExecutedFence(match, tag, inline, body) {
+  const inlineArgs = (inline || '').trim();
+  if (!inlineArgs) return '';
+  const bodyText = (body || '').trim();
+  const content = bodyText ? `${inlineArgs}\n${bodyText}` : inlineArgs;
+  try {
+    JSON.parse(content);
+  } catch {
+    return match;
+  }
+  return '';
+}
+
 async function loadExecFenceRegex() {
   try {
     const res = await fetch('/api/tools', { credentials: 'same-origin' });
@@ -434,7 +451,10 @@ async function loadExecFenceRegex() {
       .filter((id) => id && !EXEC_FENCE_NON_TOOL.has(id));
     if (tags.length) {
       EXEC_FENCE_RE = new RegExp(
-        '```(?:' + tags.join('|') + ')\\s*\\n[\\s\\S]*?```', 'gi'
+        '```(' + tags.map(escapeRegex).join('|') + ')(?![\\w-])' +
+        '[ \\t]*([\\[{][^\\n]*?)?[ \\t]*(?=\\r?\\n|```)' +
+        '\\r?\\n?([\\s\\S]*?)```',
+        'gi'
       );
     }
   } catch (err) {
@@ -889,7 +909,7 @@ export function roleTimestamp(when) {
  */
 export function stripToolBlocks(text) {
   let cleaned = text.replace(TOOL_CALL_RE, '');
-  if (EXEC_FENCE_RE) cleaned = cleaned.replace(EXEC_FENCE_RE, '');
+  if (EXEC_FENCE_RE) cleaned = cleaned.replace(EXEC_FENCE_RE, stripExecutedFence);
   cleaned = cleaned.replace(DSML_TOOL_RE, '');
   cleaned = cleaned.replace(DSML_STRAY_RE, '');
   cleaned = cleaned.replace(XML_TOOL_CALL_RE, '');
@@ -1105,6 +1125,17 @@ document.addEventListener('click', function(e) {
   }
 }, true);
 
+function resolveDocumentPlaceholderLinks(text, metadata) {
+  if (!text || !metadata || !Array.isArray(metadata.tool_events)) return text;
+  const docEvents = metadata.tool_events.filter(ev => ev && ev.doc_id);
+  if (!docEvents.length) return text;
+  return String(text).replace(/#document-(\d+)\b/g, (match, num) => {
+    const idx = Number(num) - 1;
+    const ev = Number.isInteger(idx) && idx >= 0 ? docEvents[idx] : null;
+    return ev && ev.doc_id ? `#document-${ev.doc_id}` : match;
+  });
+}
+
 // Jump-to-entity anchors — the agent emits links like
 //   [New Chat](#session-89effa28)
 //   [Notes](#document-abc123)
@@ -1726,8 +1757,9 @@ export function createUserMsgFooter(msgElement) {
  * Display performance metrics for a message.
  */
 export function displayMetrics(messageElement, metrics) {
-  const existingMetrics = messageElement.querySelector('.response-metrics');
-  if (existingMetrics) existingMetrics.remove();
+  messageElement
+    .querySelectorAll('.response-metrics, .metrics-divider, .ctx-divider, .ctx-ring')
+    .forEach((el) => el.remove());
 
   const metricsContainer = document.createElement('span');
   metricsContainer.className = 'response-metrics';
@@ -1742,7 +1774,7 @@ export function displayMetrics(messageElement, metrics) {
   const cost = _billableCost(model, inputTokens, outputTokens);
 
   // Nothing useful to show — bail out (only if ALL metrics are missing)
-  if (!responseTime && !outputTokens && tps == null && !ctxPct) return;
+  if (!responseTime && !inputTokens && !outputTokens && tps == null && !ctxPct) return;
 
   // Accumulate session cost (only on fresh metrics, not history reload)
   if (!metrics._fromHistory) {
@@ -1765,14 +1797,17 @@ export function displayMetrics(messageElement, metrics) {
       ? `${outputTokens} tok · ${costStr0}`
       : outputTokens
         ? `${outputTokens} tok · ${responseTime != null ? responseTime + 's' : ''}`
-        : responseTime != null
-          ? `${responseTime}s`
-          : '';
+        : inputTokens
+          ? `${inputTokens} in${responseTime != null ? ' · ' + responseTime + 's' : ''}`
+          : responseTime != null
+            ? `${responseTime}s`
+            : '';
   if (!metricsLabel) return;
   metricsContainer.textContent = metricsLabel;
   metricsContainer.style.cursor = 'pointer';
   metricsContainer.title = 'Click for details';
   const metricsDivider = document.createElement('span');
+  metricsDivider.className = 'metrics-divider';
   metricsDivider.textContent = ' | ';
   metricsDivider.style.color = 'var(--color-muted-alt)';
   metricsDivider.style.pointerEvents = 'none';
@@ -1985,6 +2020,13 @@ export function displayMetrics(messageElement, metrics) {
   }
 
   let footer = messageElement.querySelector('.msg-footer');
+  if (!footer) {
+    footer = createMsgFooter(messageElement);
+    if (messageElement.classList?.contains('agent-thread')) {
+      footer.classList.add('agent-thread-footer');
+    }
+    messageElement.appendChild(footer);
+  }
   if (footer) {
     const actions = footer.querySelector('.msg-actions');
     if (actions) {
@@ -2184,7 +2226,7 @@ export function addMessage(role, content, modelName, metadata) {
 
       for (let r = 0; r < maxRound; r++) {
         const roundNum = r + 1;
-        const txt = (roundTexts[r] || '').trim();
+        const txt = resolveDocumentPlaceholderLinks((roundTexts[r] || '').trim(), metadata);
 
         if (txt) {
           const wrap = document.createElement('div');
@@ -2372,6 +2414,9 @@ export function addMessage(role, content, modelName, metadata) {
     b.className = 'body';
 
     let text = markdownModule.squashOutsideCode(stripToolBlocks(textRaw || ''));
+    if (role === 'assistant') {
+      text = resolveDocumentPlaceholderLinks(text, metadata);
+    }
 
     // For user messages, pull out vision-model image descriptions ([Image: name]\n
     // <multi-line desc>) into a collapsible "image description" section. Done for
@@ -2658,6 +2703,7 @@ const chatRenderer = {
   createMsgFooter,
   displayMetrics,
   addMessage,
+  buildAttachCards,
   updateMessageAttachments,
 };
 

static/js/chatStream.js

@@ -7,6 +7,7 @@ import Storage from './storage.js';
 import themeModule from './theme.js';
 import markdownModule from './markdown.js';
 import sessionModule from './sessions.js';
+import documentModule from './document.js';
 
 /**
  * Handle a ui_control SSE event — AI-driven UI manipulation.
@@ -183,6 +184,19 @@ export function handleUIControl(uiData) {
       }
 
     } else if (uiEvent === 'open_email_reply' || uiData.ui_event === 'open_email_reply') {
+      try {
+        var existingDocId = documentModule && documentModule.findEmailDocId
+          ? documentModule.findEmailDocId(uiData.uid, uiData.folder || 'INBOX')
+          : null;
+        if (existingDocId && documentModule.replaceEmailReplyBody) {
+          if (documentModule.loadDocument) documentModule.loadDocument(existingDocId);
+          documentModule.replaceEmailReplyBody(existingDocId, uiData.body || '', { force: true });
+          if (uiModule && uiModule.showToast) uiModule.showToast('Wrote reply into the open email');
+          return;
+        }
+      } catch (e) {
+        console.warn('open_email_reply existing draft update failed:', e);
+      }
       import('./emailInbox.js').then(function(mod) {
         var fn = mod.openReplyDraft || (mod.default && mod.default.openReplyDraft);
         if (fn) fn(uiData.uid, uiData.folder || 'INBOX', uiData.mode || 'reply', uiData.body || '');

static/js/cookbook-hwfit.js

@@ -80,6 +80,7 @@ export let _cachedModelIds = null; // repo IDs already downloaded
 // after the user has switched servers.
 let _hwfitFetchToken = 0;
 let _dismissedHwChips = new Set();
+let _hwfitAutoScanStarted = new Set();
 // Permanently removed (X-clicked) chips. Separate from _dismissedHwChips
 // so the ranker treats "off" and "removed" the same (both ignore the
 // hardware) but the UI keeps "off" chips visible to toggle back on,
@@ -241,8 +242,7 @@ export function _renderGpuToggles(system) {
       container._activeCount = undefined;   // default to the new pool's max
       delete container.dataset.rendered;    // force a count-button rebuild
       _renderGpuToggles(system);
-      _hwfitCache = null;
-      _hwfitFetch();
+      _hwfitFetch(false, { keepPrevious: true, forceRevalidate: true });
     });
   }
 
@@ -274,8 +274,7 @@ export function _renderGpuToggles(system) {
           }
         }
       }
-      _hwfitCache = null;
-      _hwfitFetch();
+      _hwfitFetch(false, { keepPrevious: true, forceRevalidate: true });
     });
   }
 }
@@ -435,6 +434,27 @@ function _readScanCache(sig) {
   return null;
 }
 
+function _readNearestScanCache(sig) {
+  try {
+    const wanted = JSON.parse(sig || '{}');
+    const all = JSON.parse(localStorage.getItem(_SCAN_CACHE_KEY) || '{}');
+    let best = null;
+    for (const [key, entry] of Object.entries(all)) {
+      if (!entry || !entry.data || (Date.now() - (entry.ts || 0)) >= _SCAN_CACHE_TTL) continue;
+      let parsed = null;
+      try { parsed = JSON.parse(key); } catch { continue; }
+      if (!parsed) continue;
+      if ((parsed.h || '') !== (wanted.h || '')) continue;
+      if ((parsed.hk || '') !== (wanted.hk || '')) continue;
+      if (JSON.stringify(parsed.m || {}) !== JSON.stringify(wanted.m || {})) continue;
+      if (JSON.stringify(parsed.d || []) !== JSON.stringify(wanted.d || [])) continue;
+      if (!best || (entry.ts || 0) > (best.ts || 0)) best = entry;
+    }
+    return best?.data || null;
+  } catch {}
+  return null;
+}
+
 function _writeScanCache(sig, data) {
   try {
     const all = JSON.parse(localStorage.getItem(_SCAN_CACHE_KEY) || '{}');
@@ -468,7 +488,7 @@ function _hwfitShowError(list, host, detail) {
   if (rb) rb.addEventListener('click', () => { _resetGpuToggleState(); _hwfitFetch(true); });
 }
 
-// Client-side "Engine" filter (llama.cpp / vLLM / SGLang / Ollama). Empty =
+// Client-side "Engine" filter (llama.cpp / vLLM / SGLang / Ollama / Diffusers). Empty =
 // show all. Uses the same _detectBackend() the serve commands use, so what you
 // filter to is exactly what would be launched. Pure view filter — no refetch
 // needed. Ollama rows are merged into the main list (see _ensureOllamaLib +
@@ -561,8 +581,11 @@ function _ollamaToHwfitRows(libModels, vramAvail, ramAvail) {
   return out;
 }
 
-export async function _hwfitFetch(fresh = false) {
+export async function _hwfitFetch(fresh = false, opts = {}) {
   const _tk = ++_hwfitFetchToken;
+  const allowNetwork = fresh || opts.allowNetwork !== false;
+  const keepPrevious = !!opts.keepPrevious;
+  const forceRevalidate = !!opts.forceRevalidate;
   const useCase = document.getElementById('hwfit-usecase')?.value || '';
   const search = document.getElementById('hwfit-search')?.value?.trim() || '';
   const remoteHost = _envState.remoteHost || '';
@@ -575,8 +598,12 @@ export async function _hwfitFetch(fresh = false) {
   // reload shows the last result with no spinner. We still fetch fresh below and
   // swap it in. If there's no cache hit, fall back to the spinner.
   const _sig = _scanSig();
-  const _cached = fresh ? null : _readScanCache(_sig);
+  let _cached = fresh ? null : _readScanCache(_sig);
+  if (!_cached && !fresh && (!allowNetwork || keepPrevious)) {
+    _cached = _readNearestScanCache(_sig);
+  }
   const wp = spinnerModule.createWhirlpool(18);
+  const _paintedFromCache = !!_cached;
   if (_cached) {
     // Tag the restored cache with its host too (scan-sig keys cache per
     // host, so a hit here is always for the current remoteHost).
@@ -587,28 +614,64 @@ export async function _hwfitFetch(fresh = false) {
     }
     _hwfitRenderList(list, _applyEngineFilter(_cached.models));
   } else {
-    // Show spinner while scanning — stack the spinner above a text label
-    // (the .hwfit-loading class is a centered flex ROW, so force column here).
-    const loadingDiv = document.createElement('div');
-    loadingDiv.className = 'hwfit-loading';
-    loadingDiv.style.flexDirection = 'column';
-    loadingDiv.style.gap = '6px';
-    loadingDiv.appendChild(wp.element);
-    // Text label like the other cookbook tabs: "Loading…", then if the scan runs
-    // long (remote SSH hardware probe), switch to "Scanning hardware…".
-    const loadingLbl = document.createElement('div');
-    loadingLbl.textContent = 'Loading…';
-    loadingLbl.style.cssText = 'text-align:center;opacity:0.5;font-size:11px;';
-    loadingDiv.appendChild(loadingLbl);
-    setTimeout(() => { if (loadingLbl.isConnected) loadingLbl.textContent = 'Scanning hardware…'; }, 2000);
-    list.innerHTML = '';
-    list.appendChild(loadingDiv);
-    _hwfitCache = null;   // no instant paint — clear until the fetch returns
+    const canKeepPrevious = keepPrevious && _hwfitCache && Array.isArray(_hwfitCache.models);
+    if (canKeepPrevious) {
+      try { wp.destroy(); } catch {}
+    } else if (!allowNetwork) {
+      _hwfitCache = null;
+      _hwfitRenderHw(hw, null);
+      const loadingDiv = document.createElement('div');
+      loadingDiv.className = 'hwfit-loading';
+      loadingDiv.style.cssText = 'flex-direction:column;gap:6px;text-align:center;';
+      loadingDiv.appendChild(wp.element);
+      const loadingTitle = document.createElement('div');
+      loadingTitle.textContent = 'No cached scan yet';
+      loadingTitle.style.cssText = 'font-size:12px;opacity:0.7;';
+      const loadingLbl = document.createElement('div');
+      loadingLbl.textContent = 'Scanning hardware…';
+      loadingLbl.style.cssText = 'font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;';
+      loadingDiv.appendChild(loadingTitle);
+      loadingDiv.appendChild(loadingLbl);
+      list.innerHTML = '';
+      list.appendChild(loadingDiv);
+      if (!_hwfitAutoScanStarted.has(_sig)) {
+        _hwfitAutoScanStarted.add(_sig);
+        setTimeout(() => {
+          if (_tk === _hwfitFetchToken) {
+            _resetGpuToggleState();
+            _hwfitFetch(true, { autoFromEmpty: true });
+          }
+        }, 60);
+      }
+      return;
+    }
+    if (!canKeepPrevious) {
+      // Show spinner while scanning — stack the spinner above a text label
+      // (the .hwfit-loading class is a centered flex ROW, so force column here).
+      const loadingDiv = document.createElement('div');
+      loadingDiv.className = 'hwfit-loading';
+      loadingDiv.style.flexDirection = 'column';
+      loadingDiv.style.gap = '6px';
+      loadingDiv.appendChild(wp.element);
+      // Text label like the other cookbook tabs: "Loading…", then if the scan runs
+      // long (remote SSH hardware probe), switch to "Scanning hardware…".
+      const loadingLbl = document.createElement('div');
+      loadingLbl.textContent = 'Loading…';
+      loadingLbl.style.cssText = 'text-align:center;opacity:0.5;font-size:11px;';
+      loadingDiv.appendChild(loadingLbl);
+      setTimeout(() => { if (loadingLbl.isConnected) loadingLbl.textContent = 'Scanning hardware…'; }, 2000);
+      list.innerHTML = '';
+      list.appendChild(loadingDiv);
+      _hwfitCache = null;   // no instant paint — clear until the fetch returns
+    }
+  }
+  if (!allowNetwork) {
+    try { wp.destroy(); } catch {}
+    return;
   }
   // Only fetch cached model IDs when server changes, not on every search/sort
   const remoteKey = _currentServerValue();
   if (!_cachedModelIds || _lastCacheHost() !== remoteKey) {
-    _setLastCacheHost(remoteKey);
     const _cacheSrv = _serverByVal(_envState.remoteServerKey || remoteHost);
     const _cachePort = _cacheSrv?.port || '';
     const _cacheParams = new URLSearchParams();
@@ -620,9 +683,11 @@ export async function _hwfitFetch(fresh = false) {
     fetch(`/api/model/cached?${_cacheParams}`, { credentials: 'same-origin' })
       .then(r => r.json())
       .then(d => {
+        if (d && d.error) throw new Error(d.error);
         // Exclude stalled (download-shell) entries — a 12 KB README-only
         // folder shouldn't count as "downloaded" in the Scan/Download list.
         _cachedModelIds = new Set((d.models || []).filter(m => m.status !== 'stalled').map(m => m.repo_id));
+        _setLastCacheHost(remoteKey);
         // Re-mark rows if already rendered
         list.querySelectorAll('.hwfit-row[data-model]').forEach(row => {
           const name = row.dataset.model;
@@ -633,7 +698,14 @@ export async function _hwfitFetch(fresh = false) {
             }
           }
         });
-      }).catch(() => {});
+      }).catch((err) => {
+        console.warn('Cached model marker scan failed:', err);
+        _setLastCacheHost('');
+      });
+  }
+  if (_paintedFromCache && !forceRevalidate) {
+    try { wp.destroy(); } catch {}
+    return;
   }
   try {
     const sortBy = document.getElementById('hwfit-sort')?.value || 'newest';
@@ -857,6 +929,8 @@ function _renderHwVisibilityWarning(sys) {
   box.querySelector('[data-hw-action="manual"]')?.addEventListener('click', () => {
     const panel = document.getElementById('hwfit-manual-panel');
     if (panel) panel.classList.remove('hidden');
+    const manualBtn = document.getElementById('hwfit-hw-manual-btn');
+    if (manualBtn) manualBtn.textContent = 'CANCEL';
     document.getElementById('hwfit-hw-manual-btn')?.scrollIntoView?.({
       behavior: 'smooth',
       block: 'center',
@@ -1021,6 +1095,8 @@ export function _hwfitRenderHw(el, sys) {
         _saveManualHwState(null);
         btn.closest('.hwfit-hw-chip-row')?.remove();
         document.getElementById('hwfit-manual-panel')?.classList.add('hidden');
+        const manualBtn = document.getElementById('hwfit-hw-manual-btn');
+        if (manualBtn) manualBtn.textContent = 'EDIT';
         _resetGpuToggleState();
         _hwfitCache = null;
         _hwfitFetch(true);
@@ -1041,16 +1117,20 @@ function _wireManualHardwareControls(el) {
   const btn = document.getElementById('hwfit-hw-manual-btn');
   const panel = document.getElementById('hwfit-manual-panel');
   if (!btn || !panel) return;
+  const syncManualButton = () => {
+    btn.textContent = panel.classList.contains('hidden') ? 'EDIT' : 'CANCEL';
+  };
   const clearManual = () => {
     _saveManualHwState(null);
     el.querySelector('.hwfit-hw-chip-manual')?.remove();
     panel.classList.add('hidden');
+    syncManualButton();
     _resetGpuToggleState();
     _hwfitCache = null;
     _hwfitFetch(true);
   };
   const manual = _manualHwState();
-  btn.textContent = 'EDIT';
+  syncManualButton();
   if (manual) {
     panel.querySelector('.hwfit-manual-mode').value = manual.mode || 'gpu';
     panel.querySelector('.hwfit-manual-backend').value = manual.backend || 'cuda';
@@ -1066,11 +1146,13 @@ function _wireManualHardwareControls(el) {
     btn._hwfitManualBound = true;
     btn.addEventListener('click', () => {
       panel.classList.toggle('hidden');
+      syncManualButton();
       syncMode();
     });
   }
   el.querySelector('.hwfit-hw-chip-toggle[data-hw-chip="manual"]')?.addEventListener('click', () => {
     panel.classList.remove('hidden');
+    syncManualButton();
     syncMode();
   });
   if (!panel._hwfitManualBound) {
@@ -1087,12 +1169,14 @@ function _wireManualHardwareControls(el) {
       _resetGpuToggleState();
       _hwfitCache = null;
       panel.classList.add('hidden');
+      syncManualButton();
       _hwfitRenderHw(el, _manualDisplaySystem(window._hwfitSystemCache, manual));
       _hwfitFetch(true);
     });
     panel.querySelector('.hwfit-hw-manual-clear')?.addEventListener('click', clearManual);
   }
   syncMode();
+  syncManualButton();
 }
 
 export const _fitColors = { perfect: 'var(--green, #50fa7b)', good: 'var(--yellow, #f1fa8c)', marginal: 'var(--orange, #ffb86c)', too_tight: 'var(--red, #ff5555)' };
@@ -1780,6 +1864,83 @@ export function _expandModelRow(row, modelData) {
 
 }
 
+const _HWFIT_ENGINE_GLYPHS = {
+  '': '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><line x1="4" y1="6" x2="20" y2="6"></line><line x1="4" y1="12" x2="20" y2="12"></line><line x1="4" y1="18" x2="20" y2="18"></line><circle cx="8" cy="6" r="2" fill="currentColor" stroke="none"></circle><circle cx="16" cy="12" r="2" fill="currentColor" stroke="none"></circle><circle cx="10" cy="18" r="2" fill="currentColor" stroke="none"></circle></svg>',
+  vllm: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"></path><path d="M14 4l4 9 3-9"></path></svg>',
+  sglang: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;"></span>',
+  llamacpp: '<svg width="14" height="14" viewBox="0 0 600 600" fill="none" aria-hidden="true"><path d="M600 392L504.249 558L504.137 557.929C487.252 584.069 458.193 600 426.864 600H120L240 392H600Z" fill="currentColor"></path><path d="M240 392H0L199.602 46.0254C216.032 17.5463 246.411 0 279.29 0H466.154L240 392Z" fill="currentColor"></path></svg>',
+  ollama: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;"></span>',
+  diffusers: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"></circle><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"></path></svg>',
+};
+
+function _hwfitEngineGlyph(value) {
+  return _HWFIT_ENGINE_GLYPHS[value] || _HWFIT_ENGINE_GLYPHS[''];
+}
+
+function _bindHwfitEnginePicker(engine) {
+  const wrap = engine?.closest('.hwfit-engine-wrap');
+  const btn = wrap?.querySelector('[data-hwfit-engine-btn]');
+  const menu = wrap?.querySelector('[data-hwfit-engine-menu]');
+  const icon = wrap?.querySelector('[data-hwfit-engine-icon]');
+  const label = wrap?.querySelector('[data-hwfit-engine-label]');
+  if (!engine || !wrap || !btn || !menu || wrap.dataset.enginePickerBound) return;
+  wrap.dataset.enginePickerBound = '1';
+
+  const setOpen = (open) => {
+    menu.hidden = !open;
+    btn.setAttribute('aria-expanded', open ? 'true' : 'false');
+  };
+  const currentLabel = () => {
+    const opt = Array.from(engine.options).find((o) => o.value === engine.value);
+    return opt?.textContent || 'Engine';
+  };
+  const syncButton = () => {
+    if (label) label.textContent = currentLabel();
+    if (icon) icon.innerHTML = _hwfitEngineGlyph(engine.value);
+    menu.querySelectorAll('[data-hwfit-engine-value]').forEach((item) => {
+      const active = item.dataset.hwfitEngineValue === engine.value;
+      item.classList.toggle('active', active);
+      item.setAttribute('aria-selected', active ? 'true' : 'false');
+    });
+  };
+  const renderMenu = () => {
+    menu.innerHTML = Array.from(engine.options).map((opt) => (
+      `<button type="button" role="option" class="hwfit-engine-item" data-hwfit-engine-value="${opt.value}">`
+      + `<span class="hwfit-engine-item-icon" aria-hidden="true">${_hwfitEngineGlyph(opt.value)}</span>`
+      + `<span class="hwfit-engine-item-label">${opt.textContent}</span>`
+      + '</button>'
+    )).join('');
+    menu.querySelectorAll('[data-hwfit-engine-value]').forEach((item) => {
+      item.addEventListener('click', (ev) => {
+        ev.preventDefault();
+        ev.stopPropagation();
+        const next = item.dataset.hwfitEngineValue || '';
+        if (engine.value !== next) {
+          engine.value = next;
+          engine.dispatchEvent(new Event('change', { bubbles: true }));
+        }
+        syncButton();
+        setOpen(false);
+      });
+    });
+    syncButton();
+  };
+
+  btn.addEventListener('click', (ev) => {
+    ev.preventDefault();
+    ev.stopPropagation();
+    setOpen(menu.hidden);
+  });
+  engine.addEventListener('change', syncButton);
+  document.addEventListener('click', (ev) => {
+    if (!wrap.contains(ev.target)) setOpen(false);
+  });
+  document.addEventListener('keydown', (ev) => {
+    if (ev.key === 'Escape') setOpen(false);
+  });
+  renderMenu();
+}
+
 export function _hwfitInit() {
   const uc = document.getElementById('hwfit-usecase');
   const sort = document.getElementById('hwfit-sort');
@@ -1795,6 +1956,7 @@ export function _hwfitInit() {
   // Engine filter is a pure client-side view filter over the already-fetched
   // list (HF + Ollama merged), so just re-render from cache.
   const engine = document.getElementById('hwfit-engine');
+  if (engine) _bindHwfitEnginePicker(engine);
   if (engine) engine.addEventListener('change', () => {
     const list = document.getElementById('hwfit-list');
     if (list && _hwfitCache && Array.isArray(_hwfitCache.models)) {
@@ -1955,16 +2117,17 @@ export function _hwfitInit() {
     dot.className = 'cookbook-srv-status testing';
     dot.title = 'Testing SSH…';
     setMsg('Testing SSH...');
-    const pf = port && port !== '22' ? `-p ${port} ` : '';
-    const cmd = `ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ${pf}${host} "echo ok"`;
     const t0 = Date.now();
     try {
-      const res = await fetch('/api/shell/exec', {
+      const res = await fetch('/api/cookbook/test-ssh', {
         method: 'POST', credentials: 'same-origin',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ command: cmd, timeout: 8 }),
+        body: JSON.stringify({ host, ssh_port: port || undefined }),
       });
       const data = await res.json();
+      if (!res.ok) {
+        throw new Error(data.detail || data.error || `HTTP ${res.status}`);
+      }
       const ms = Date.now() - t0;
       const out = (data.stdout || '').trim();
       if (data.exit_code === 0 && out.startsWith('ok')) {
@@ -1973,7 +2136,7 @@ export function _hwfitInit() {
         setMsg(`Connected · ${ms} ms`, 'var(--green,#50fa7b)');
       } else {
         dot.className = 'cookbook-srv-status fail';
-        const err = (data.stderr || data.stdout || `exit ${data.exit_code}`).toString().trim().slice(0, 240);
+        const err = (data.stderr || data.stdout || (data.exit_code == null ? 'no exit code' : `exit ${data.exit_code}`)).toString().trim().slice(0, 240);
         dot.title = `SSH failed: ${err}`;
         setMsg(`Failed · ${err}`, 'var(--red,#e06c75)');
       }
@@ -2159,15 +2322,12 @@ export function _hwfitInit() {
         }
       });
     });
-    // Auto-test when host or port blur
+    // Manual connectivity test after editing host or port. Existing saved
+    // servers are not auto-tested on panel open; unreachable hosts can stall the
+    // Cookbook UI and make opening the panel feel blocked.
     entry.querySelectorAll('.cookbook-srv-host, .cookbook-srv-port').forEach(el => {
       el.addEventListener('blur', () => _testServerConnection(entry));
     });
-    // Initial test for pre-filled rows (existing servers on tab load)
-    if (entry.querySelector('.cookbook-srv-host')?.value?.trim() && !entry.dataset.tested) {
-      entry.dataset.tested = '1';
-      _testServerConnection(entry);
-    }
     // Cancel button on a brand-new server entry: discard it (no confirm — it's
     // unsaved) and re-sync so the dropped blank server doesn't linger.
     const cancelBtn = entry.querySelector('.cookbook-server-cancel-btn');

static/js/cookbook.js

@@ -205,7 +205,7 @@ export function _sshCmd(host, cmd, port) {
 /** Get SSH port for a given host (or task object) */
 function _getPort(hostOrTask) {
   if (!hostOrTask) return '';
-  if (typeof hostOrTask === 'object') return hostOrTask.sshPort || _getPort(hostOrTask.remoteServerKey || hostOrTask.remoteHost);
+  if (typeof hostOrTask === 'object') return hostOrTask.sshPort || _getPort(hostOrTask.remoteServerKey || hostOrTask.remoteHost || hostOrTask.payload?.remote_host);
   const selected = hostOrTask === _envState.remoteHost ? _selectedServer() : null;
   const srv = selected || _serverByVal(hostOrTask);
   return srv?.port || '';
@@ -891,8 +891,9 @@ async function _fetchDependencies() {
   let _spin = null;
   try {
     const sp = (await import('./spinner.js')).default;
-    _spin = sp.createWhirlpool(28);
-    _spin.element.style.cssText = 'margin:24px auto 0;display:block;';
+    _spin = sp.createWhirlpool(22);
+    _spin.element.classList.add('cookbook-section-loading-wp');
+    _spin.element.style.cssText = 'margin:24px auto 0;display:block;width:22px;height:22px;';
     list.appendChild(_spin.element);
     const label = document.createElement('div');
     label.className = 'hwfit-loading';
@@ -1652,10 +1653,10 @@ function _wireTabEvents(body) {
       });
       if (backend === 'Search') {
         _hwfitInit();
-        _hwfitFetch();
+        _hwfitFetch(false, { allowNetwork: false });
       }
       if (backend === 'Serve') {
-        _fetchCachedModels();
+        _fetchCachedModels(false, { allowNetwork: false });
       }
       if (backend === 'Dependencies') {
         _fetchDependencies();
@@ -1758,7 +1759,7 @@ function _wireTabEvents(body) {
       _applyServerSelection(dlServer.value);
       // Reset toggle state (no flicker) so the new server's hardware re-renders.
       _resetGpuToggleState();
-      _hwfitFetch();
+      _hwfitFetch(false, { allowNetwork: false });
     });
   }
 
@@ -1795,13 +1796,13 @@ function _wireTabEvents(body) {
           if (settingsTab) settingsTab.click();
         });
       }
-      _fetchCachedModels();
+      _fetchCachedModels(false, { allowNetwork: false });
     });
   }
 
   const scanBtn = document.getElementById('hwfit-cache-scan');
   if (scanBtn) {
-    scanBtn.addEventListener('click', () => _fetchCachedModels());
+    scanBtn.addEventListener('click', () => _fetchCachedModels(true));
   }
 
   const editDirsLink = document.querySelector('.cookbook-serve-dir-edit');
@@ -2299,8 +2300,9 @@ function _wireTabEvents(body) {
       hfList.innerHTML = '';
       try {
         const sp = (await import('./spinner.js')).default;
-        const _spin = sp.createWhirlpool(28);
-        _spin.element.style.cssText = 'margin:24px auto 0;display:block;';
+        const _spin = sp.createWhirlpool(22);
+        _spin.element.classList.add('cookbook-section-loading-wp');
+        _spin.element.style.cssText = 'margin:24px auto 0;display:block;width:22px;height:22px;';
         hfList.appendChild(_spin.element);
         const lbl = document.createElement('div');
         lbl.className = 'hwfit-loading';
@@ -2522,7 +2524,7 @@ function _wireTabEvents(body) {
 // (a new server's host is empty, which would otherwise read as "Local").
 export function _serverEntryHtml(s, i, defaultServer, forceRemote, isNew) {
   const isLocal = (forceRemote || isNew) ? false : (!s.host || s.host === 'local');
-  const envOpts = ['none', 'venv'].map(e => `<option value="${e}"${s.env === e ? ' selected' : ''}>${e === 'none' ? 'None' : e}</option>`).join('');
+  const envOpts = [['none', 'None'], ['venv', 'venv'], ['conda', 'conda']].map(([value, label]) => `<option value="${value}"${s.env === value ? ' selected' : ''}>${label}</option>`).join('');
   let html = '';
   html += `<div class="cookbook-server-entry" data-idx="${i}" data-platform="${esc(s.platform || '')}">`;
   const _srvTitle = s.name || (isLocal ? 'Local' : (s.host || `Server ${i + 1}`));
@@ -2548,7 +2550,7 @@ export function _serverEntryHtml(s, i, defaultServer, forceRemote, isNew) {
   html += `<input type="text" class="hwfit-sf cookbook-srv-host" value="${isLocal ? '' : esc(s.host || '')}" placeholder="e.g. user@ip" style="width:214.5px;flex-shrink:0;box-sizing:border-box;" ${isLocal ? 'readonly' : ''} />`;
   html += `<input type="text" class="hwfit-sf cookbook-srv-port" value="${esc(s.port || '')}" placeholder="Port" title="SSH port (default 22)" style="width:48px;flex-shrink:0;" ${isLocal ? 'readonly' : ''} />`;
   html += `<select class="hwfit-sf cookbook-srv-env">${envOpts}</select>`;
-  html += `<input type="text" class="hwfit-sf cookbook-srv-path" value="${esc(s.envPath || '')}" placeholder="${s.platform === 'windows' ? 'venv path' : '~/venv'}" />`;
+  html += `<input type="text" class="hwfit-sf cookbook-srv-path" value="${esc(s.envPath || '')}" placeholder="${s.platform === 'windows' ? 'venv/conda env' : '~/venv or conda-env'}" />`;
   html += `<span class="cookbook-dep-tag cookbook-dep-target" style="font-size:8px;flex-shrink:0;min-width:46px;text-align:center;visibility:hidden;">placeholder</span>`;
   html += `<span class="cookbook-srv-actions" style="display:inline-flex;gap:4px;align-items:center;width:78px;flex-shrink:0;justify-content:flex-end;"></span>`;
   html += `</div>`;
@@ -2694,8 +2696,7 @@ function _renderRecipes() {
   html += '<p class="memory-desc doclib-desc" style="margin-top:6px;">Scans your hardware for what models you can run. Hardware is cached; hit the scan button to re-probe after changing GPUs.</p>';
   html += '<div class="hwfit-toolbar" style="margin-top:9px;">';
   html += '<select class="cookbook-field-input hwfit-usecase" id="hwfit-usecase" style="height:28px;">';
-  html += '<option value="general" selected>Standard</option><option value="coding">Coding</option>';
-  html += '<option value="reasoning">Reasoning</option><option value="chat">Chat</option>';
+  html += '<option value="general" selected>Standard</option>';
   // Image tab removed — text→image gen is gone from this build (only inpaint
    // remains, which uses its own settings panel). Vision (multimodal) stays.
   html += '<option value="multimodal">Vision</option></select>';
@@ -2704,13 +2705,20 @@ function _renderRecipes() {
   // levers (Engine / Quant / Context) live to the right.
   html += '<input type="text" class="cookbook-field-input hwfit-search" id="hwfit-search" placeholder="Search models..." style="flex:1;" />';
   html += '<span class="hwfit-engine-wrap">';
-  html += '<select class="cookbook-field-input hwfit-engine" id="hwfit-engine" style="height:28px;" title="Filter by serving engine">';
+  html += '<select class="cookbook-field-input hwfit-engine" id="hwfit-engine" style="display:none;" title="Filter by serving engine">';
   html += '<option value="">Engine</option>';
   html += '<option value="llamacpp">llama.cpp</option>';
   html += '<option value="ollama">Ollama</option>';
   html += '<option value="vllm">vLLM</option>';
   html += '<option value="sglang">SGLang</option>';
+  html += '<option value="diffusers">Diffusers</option>';
   html += '</select>';
+  html += '<button type="button" class="cookbook-field-input hwfit-engine-btn" data-hwfit-engine-btn aria-haspopup="listbox" aria-expanded="false" title="Filter by serving engine">';
+  html += '<span class="hwfit-engine-btn-icon" data-hwfit-engine-icon aria-hidden="true"></span>';
+  html += '<span class="hwfit-engine-btn-label" data-hwfit-engine-label>Engine</span>';
+  html += '<svg class="hwfit-engine-caret" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="6 9 12 15 18 9"></polyline></svg>';
+  html += '</button>';
+  html += '<div class="hwfit-engine-menu" data-hwfit-engine-menu role="listbox" hidden></div>';
   html += '<span class="hwfit-help-chip hwfit-help-chip-inline hwfit-engine-help" title="Rule of thumb: GGUF on single GPU / CPU+RAM → llama.cpp (or Ollama). Safetensors on multi-GPU NVIDIA → vLLM. SGLang is a vLLM-class alternative, sometimes faster on big-MoE / long-context.">?</span>';
   html += '</span>';
   // Quant (Q4/Q8/…). Default is "All" so the list shows the best-scoring
@@ -2875,7 +2883,7 @@ function _renderRecipes() {
 
   // Auto-init What Fits
   _hwfitInit();
-  _hwfitFetch();
+  _hwfitFetch(false, { allowNetwork: false });
 }
 
 // ── Public API ──
@@ -3087,8 +3095,48 @@ export function isVisible() {
 
 let _sharedSyncInFlight = false;
 let _sharedSyncLast = 0;
+const SHARED_STATE_LEADER_KEY = 'odysseus-cookbook-shared-state-leader';
+const SHARED_STATE_LEADER_ID = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
+const SHARED_STATE_LEADER_TTL_MS = 12000;
+
+function _foregroundChatBusy() {
+  try {
+    return !!window.__odysseusChatBusy || Date.now() < (window.__odysseusChatBusyUntil || 0);
+  } catch (_) {
+    return false;
+  }
+}
+
+function _claimSharedStateLeader() {
+  if (document.visibilityState !== 'visible') return false;
+  const now = Date.now();
+  try {
+    const raw = localStorage.getItem(SHARED_STATE_LEADER_KEY);
+    const current = raw ? JSON.parse(raw) : null;
+    if (
+      !current
+      || !current.id
+      || current.id === SHARED_STATE_LEADER_ID
+      || now - Number(current.ts || 0) > SHARED_STATE_LEADER_TTL_MS
+    ) {
+      localStorage.setItem(SHARED_STATE_LEADER_KEY, JSON.stringify({ id: SHARED_STATE_LEADER_ID, ts: now }));
+      return true;
+    }
+    return current.id === SHARED_STATE_LEADER_ID;
+  } catch (_) {
+    return true;
+  }
+}
+
+function _canRefreshSharedCookbookState() {
+  if (!isVisible() || _sharedSyncInFlight) return false;
+  if (document.visibilityState !== 'visible') return false;
+  if (_foregroundChatBusy()) return false;
+  return _claimSharedStateLeader();
+}
+
 async function _refreshSharedCookbookState(reason = '') {
-  if (!isVisible() || _sharedSyncInFlight) return;
+  if (!_canRefreshSharedCookbookState()) return;
   const now = Date.now();
   if (now - _sharedSyncLast < 1500) return;
   _sharedSyncInFlight = true;

static/js/cookbookRunning.js

@@ -29,7 +29,8 @@ function _statusLabel(status, type) {
 function _taskBadge(task) {
   if (task._unreachable && task.status === 'running') return { text: 'unreachable', cls: 'cookbook-task-error' };
   if (task.type === 'download' && task.status === 'running') {
-    return { text: _statusLabel(task.status, task.type), cls: 'cookbook-task-downloading' };
+    const progress = String(task.progress || '').trim();
+    return { text: progress || _statusLabel(task.status, task.type), cls: 'cookbook-task-downloading' };
   }
   if (task.type === 'serve' && task.status === 'running' && task.progress) {
     // Same green "running" pill — just with dynamic phase text, so it doesn't
@@ -368,7 +369,7 @@ const SERVE_STATE_KEY = 'cookbook-serve-state';
 
 // Polling / timeout intervals
 const TASK_POLL_INTERVAL_MS = 3000;       // delay between reconnect-loop iterations
-const BG_MONITOR_INTERVAL_MS = 5000;      // background task status poll
+const BG_MONITOR_INTERVAL_MS = 10000;     // background task status poll
 const STALE_PROGRESS_MS = 5 * 60 * 1000;  // download with no progress this long = stale
 const STARTUP_STALE_PROGRESS_MS = 45 * 1000; // 0%-forever startup stall: retry much sooner
 
@@ -763,6 +764,11 @@ function _redactStoredText(value) {
     .replace(/((?:api[_-]?key|token|authorization|password|passwd|secret)\s*[=:]\s*)(["']?)[^\s"']+/gi, '$1$2[redacted]');
 }
 
+function _isServeOutputPlaceholder(value) {
+  const text = String(value || '').trim();
+  return !text || /^Launched via agent\s+—\s+waiting for tmux output/i.test(text);
+}
+
 function _redactTaskForStorage(task) {
   if (!task || typeof task !== 'object') return task;
   const safe = { ...task };
@@ -881,18 +887,23 @@ function _animateOutThenRemove(el, sessionId) {
 
 // ── tmux / Windows session commands ──
 
+function _taskRemoteHost(task) {
+  return task?.remoteHost || task?.payload?.remote_host || '';
+}
+
 export function _tmuxCmd(task, tmuxArgs) {
   if (_isWindows(task)) {
     return _winSessionCmd(task, tmuxArgs);
   }
-  if (task.remoteHost) {
-    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux ${tmuxArgs}' 2>/dev/null`;
+  const host = _taskRemoteHost(task);
+  if (host) {
+    return `ssh ${_sshPrefix(_getPort(task))}${host} 'tmux ${tmuxArgs}' 2>/dev/null`;
   }
   return `tmux ${tmuxArgs} 2>/dev/null`;
 }
 
 function _winSessionCmd(task, tmuxArgs) {
-  const host = task.remoteHost;
+  const host = _taskRemoteHost(task);
   const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
   const sid = task.sessionId;
   const pf = _sshPrefix(_getPort(task));
@@ -924,12 +935,13 @@ function _winSessionCmd(task, tmuxArgs) {
 
 function _winPowerShellCmd(task, ps) {
   const command = `powershell -Command "${ps}"`;
-  if (!task.remoteHost) return command;
-  return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(command)}`;
+  const host = _taskRemoteHost(task);
+  if (!host) return command;
+  return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(command)}`;
 }
 
 function _winSessionStopTreePs(task) {
-  const host = task.remoteHost;
+  const host = _taskRemoteHost(task);
   const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
   const sid = task.sessionId;
   const stopTree = `function Stop-Tree([int]$Id) { Get-CimInstance Win32_Process -Filter ('ParentProcessId = ' + $Id) -ErrorAction SilentlyContinue | ForEach-Object { Stop-Tree ([int]$_.ProcessId) }; Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue }`;
@@ -943,8 +955,9 @@ export function _tmuxGracefulKill(task) {
     const ps = _winSessionStopTreePs(task);
     return _winPowerShellCmd(task, ps);
   }
-  if (task.remoteHost) {
-    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
+  const host = _taskRemoteHost(task);
+  if (host) {
+    return `ssh ${_sshPrefix(_getPort(task))}${host} 'tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
   }
   return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`;
 }
@@ -969,8 +982,9 @@ export function _tmuxForceKill(task) {
     `  done; ` +
     `fi; ` +
     `tmux kill-session -t ${sid} 2>/dev/null`;
-  if (task.remoteHost) {
-    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`;
+  const host = _taskRemoteHost(task);
+  if (host) {
+    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(inner)}`;
   }
   return inner;
 }
@@ -985,8 +999,9 @@ export function _tmuxIsAliveCheck(task) {
   }
   const sid = task.sessionId;
   const inner = `if tmux has-session -t ${sid} 2>/dev/null; then echo ALIVE; else echo DEAD; fi`;
-  if (task.remoteHost) {
-    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`;
+  const host = _taskRemoteHost(task);
+  if (host) {
+    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(inner)}`;
   }
   return inner;
 }
@@ -1021,8 +1036,9 @@ function _ollamaUnloadCommand(task, outputText = '') {
   const base = _ollamaBaseUrlForTask(task, outputText);
   const body = JSON.stringify({ model, prompt: '', keep_alive: 0, stream: false });
   const inner = `curl -sf -X POST ${_shQuote(base + '/api/generate')} -H 'Content-Type: application/json' -d ${_shQuote(body)} >/dev/null 2>&1 || true`;
-  if (task.remoteHost) {
-    return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`;
+  const host = _taskRemoteHost(task);
+  if (host) {
+    return `ssh ${_sshPrefix(_getPort(task))}${host} ${_shQuote(inner)}`;
   }
   return inner;
 }
@@ -1031,7 +1047,7 @@ function _endpointUrlForTask(task, outputText = '') {
   if (_taskLooksOllama(task, outputText)) {
     return _ollamaBaseUrlForTask(task, outputText) + '/v1';
   }
-  const host = _connectHostFromRemote(task.remoteHost);
+  const host = _connectHostFromRemote(_taskRemoteHost(task));
   const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
   const port = portMatch ? portMatch[1] : '8000';
   return `http://${host}:${port}/v1`;
@@ -1859,6 +1875,7 @@ export function _renderRunningTab() {
       body.querySelectorAll('.cookbook-group').forEach(g => {
         g.classList.toggle('hidden', g.dataset.backendGroup !== 'Running');
       });
+      setTimeout(() => _renderRunningTab(), 0);
     });
   } else if (runTab) {
     const _errCount2 = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
@@ -2105,6 +2122,12 @@ export function _renderRunningTab() {
       }
       const startNow = el.querySelector('.cookbook-task-start-now');
       if (startNow) startNow.style.display = (task.type === 'download' && task.status === 'queued') ? '' : 'none';
+      const pre = el.querySelector('.cookbook-output-pre');
+      if (pre && typeof task.output === 'string' && task.output && pre.textContent !== task.output) {
+        const atBottom = (pre.scrollHeight - pre.scrollTop - pre.clientHeight) < 40;
+        pre.textContent = task.output;
+        if (atBottom) pre.scrollTop = pre.scrollHeight;
+      }
       const terminalDiag = _terminalServeDiagnosis(task, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
       if (terminalDiag) {
         _showDiagnosis(el, terminalDiag, el.querySelector('.cookbook-output-pre')?.textContent || task.output || '');
@@ -2709,7 +2732,7 @@ export function _renderRunningTab() {
     // responds; without this, the user opens the Running tab and sees
     // only the placeholder ("Launched by scheduled task …") because
     // _reconnectTask never fires for status 'ready'/'loading'/'warming'.
-    if (['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status)) {
+    if (_isRunningTabVisible() && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status)) {
       _reconnectTask(el, task);
     }
   }
@@ -2755,7 +2778,7 @@ async function _reconnectTask(el, task) {
       const res = await fetch('/api/shell/exec', {
         method: 'POST', credentials: 'same-origin',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ command: _tmuxCmd(task, `capture-pane -t ${task.sessionId} -p -S -200`), timeout: 15 }),
+        body: JSON.stringify({ command: _tmuxCmd(task, `capture-pane -t ${task.sessionId} -p -S -500`), timeout: 15 }),
       });
       const data = await res.json();
 
@@ -3434,68 +3457,144 @@ async function _reconnectTask(el, task) {
 // ── Background monitor ──
 
 let _bgMonitorInterval = null;
+let _bgPollInFlight = false;
+const BG_LEADER_KEY = 'odysseus-cookbook-bg-leader';
+const BG_LEADER_ID = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
+const BG_LEADER_TTL_MS = 15000;
+
+function _hasLiveTasks(tasks = null) {
+  const list = tasks || _loadTasks();
+  return list.some(t =>
+    t.status === 'running'
+    || t.status === 'queued'
+    || t.status === 'ready'
+    || _downloadOutputLooksActive(t)
+  );
+}
+
+function _isRunningTabVisible() {
+  const modal = document.getElementById('cookbook-modal');
+  if (!modal || modal.classList.contains('hidden')) return false;
+  const activeTab = modal.querySelector('.cookbook-tab.active')?.dataset?.backend || '';
+  return activeTab === 'Running';
+}
+
+function _foregroundChatBusy() {
+  try {
+    return !!window.__odysseusChatBusy || Date.now() < (window.__odysseusChatBusyUntil || 0);
+  } catch {
+    return false;
+  }
+}
+
+function _claimBackgroundLeader() {
+  if (document.visibilityState !== 'visible') return false;
+  const now = Date.now();
+  try {
+    const raw = localStorage.getItem(BG_LEADER_KEY);
+    const current = raw ? JSON.parse(raw) : null;
+    if (
+      !current
+      || !current.id
+      || current.id === BG_LEADER_ID
+      || now - Number(current.ts || 0) > BG_LEADER_TTL_MS
+    ) {
+      localStorage.setItem(BG_LEADER_KEY, JSON.stringify({ id: BG_LEADER_ID, ts: now }));
+      return true;
+    }
+    return current.id === BG_LEADER_ID;
+  } catch (_) {
+    return true;
+  }
+}
+
+function _canBackgroundPoll() {
+  if (_foregroundChatBusy()) return false;
+  if (document.visibilityState !== 'visible') return false;
+  return _claimBackgroundLeader();
+}
 
 // Reachability check for running serve tasks. The tmux pane can stay alive
 // while the model server inside it has crashed (so no "Process exited" line
 // ever appears) — leaving the card showing "running" forever. So we actively
 // probe the registered endpoint (same /probe-local the model picker uses) and
 // flag the card "unreachable" (red) when the server stops answering.
+let _serveReachabilityInFlight = false;
+let _serveReachabilityLastAt = 0;
 async function _checkServeReachability() {
+  // This reaches out to local model servers. Keep it out of the normal chat
+  // path unless the user is actively looking at the Running tab.
+  if (_foregroundChatBusy()) return;
+  if (!_isRunningTabVisible()) return;
+  const now = Date.now();
+  if (_serveReachabilityInFlight || now - _serveReachabilityLastAt < 10000) return;
+  _serveReachabilityInFlight = true;
+  _serveReachabilityLastAt = now;
   let serveTasks;
   try {
     serveTasks = _loadTasks().filter(t => t.type === 'serve' && t.status === 'running');
-  } catch { return; }
-  if (!serveTasks.length) return;
+  } catch {
+    _serveReachabilityInFlight = false;
+    return;
+  }
+  if (!serveTasks.length) {
+    _serveReachabilityInFlight = false;
+    return;
+  }
   let eps = [], probe = {};
   try {
     [eps, probe] = await Promise.all([
       fetch('/api/model-endpoints', { credentials: 'same-origin' }).then(r => r.json()).catch(() => []),
       fetch('/api/model-endpoints/probe-local', { credentials: 'same-origin' }).then(r => r.json()).catch(() => ({})),
     ]);
-  } catch { return; }
-  for (const task of serveTasks) {
-    const host = _connectHostFromRemote(task.remoteHost);
-    const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
-    const port = portMatch ? portMatch[1] : '8000';
-    const baseUrl = `http://${host}:${port}/v1`;
-    const ep = (eps || []).find(e => e.base_url === baseUrl);
-    if (!ep) continue;                       // not registered yet — can't judge
-    const pr = probe[ep.id];
-    if (!pr || pr.alive === undefined) continue;  // not probed (non-local) — skip
-    // Record the first time it actually answers. Until then the server is still
-    // LOADING/warming (the endpoint can get registered on the 300s timeout for a
-    // big model that hasn't finished loading), and a not-yet-answering server is
-    // not "unreachable" — flagging it as such while you're launching is a false
-    // alarm. Only treat it as unreachable once it has been reachable at least once.
-    if (pr.alive === true && !task._everReachable) {
-      task._everReachable = true;
-      _updateTask(task.sessionId, { _everReachable: true });
-    }
-    const unreachable = pr.alive === false;
-    if (unreachable && !task._everReachable) continue;  // still coming up, not crashed
-    if (!!task._unreachable !== unreachable) {
-      _updateTask(task.sessionId, { _unreachable: unreachable });
-    }
-    const el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
-    if (el) {
-      el.classList.toggle('cookbook-task-unreachable', unreachable);
-      const badge = el.querySelector('.cookbook-task-status');
-      if (badge) {
-        if (unreachable) {
-          badge.textContent = 'unreachable';
-          badge.className = 'cookbook-task-status cookbook-task-error';
-          badge.title = pr.error || 'Server not responding — it may have crashed';
-        } else if (badge.textContent === 'unreachable') {
-          // Recovered — restore the normal running label.
-          badge.textContent = _statusLabel('running', task.type);
-          badge.className = 'cookbook-task-status cookbook-task-running';
-          badge.title = '';
+    for (const task of serveTasks) {
+      const host = _connectHostFromRemote(task.remoteHost);
+      const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
+      const port = portMatch ? portMatch[1] : '8000';
+      const baseUrl = `http://${host}:${port}/v1`;
+      const ep = (eps || []).find(e => e.base_url === baseUrl);
+      if (!ep) continue;                       // not registered yet — can't judge
+      const pr = probe[ep.id];
+      if (!pr || pr.alive === undefined) continue;  // not probed (non-local) — skip
+      // Record the first time it actually answers. Until then the server is still
+      // LOADING/warming (the endpoint can get registered on the 300s timeout for a
+      // big model that hasn't finished loading), and a not-yet-answering server is
+      // not "unreachable" — flagging it as such while you're launching is a false
+      // alarm. Only treat it as unreachable once it has been reachable at least once.
+      if (pr.alive === true && !task._everReachable) {
+        task._everReachable = true;
+        _updateTask(task.sessionId, { _everReachable: true });
+      }
+      const unreachable = pr.alive === false;
+      if (unreachable && !task._everReachable) continue;  // still coming up, not crashed
+      if (!!task._unreachable !== unreachable) {
+        _updateTask(task.sessionId, { _unreachable: unreachable });
+      }
+      const el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`);
+      if (el) {
+        el.classList.toggle('cookbook-task-unreachable', unreachable);
+        const badge = el.querySelector('.cookbook-task-status');
+        if (badge) {
+          if (unreachable) {
+            badge.textContent = 'unreachable';
+            badge.className = 'cookbook-task-status cookbook-task-error';
+            badge.title = pr.error || 'Server not responding — it may have crashed';
+          } else if (badge.textContent === 'unreachable') {
+            // Recovered — restore the normal running label.
+            badge.textContent = _statusLabel('running', task.type);
+            badge.className = 'cookbook-task-status cookbook-task-running';
+            badge.title = '';
+          }
         }
       }
+      if (unreachable) _showCookbookNotif(true);
     }
-    if (unreachable) _showCookbookNotif(true);
+    _refreshServerDots();
+  } catch {
+    // Non-fatal: the normal task status poll continues separately.
+  } finally {
+    _serveReachabilityInFlight = false;
   }
-  _refreshServerDots();
 }
 
 function _serveTaskFailed(task) {
@@ -3647,16 +3746,21 @@ export async function _selfHealStaleTasks(opts = {}) {
 export function _startBackgroundMonitor() {
   if (_bgMonitorInterval) return;
   _bgMonitorInterval = setInterval(() => {
+    if (!_canBackgroundPoll()) return;
     _pollBackgroundStatus();
     _checkServeReachability();
     // Auto-reconnect: every cycle, look for download tasks marked finished/
     // crashed/etc. whose tmux session is actually still running, and flip
     // them back to running. Internally throttled to 8s so a manual call from
     // the open path or a fast invocation doesn't double up.
-    _selfHealStaleTasks().catch(() => {});
+    if (_hasLiveTasks() || _isRunningTabVisible()) {
+      _selfHealStaleTasks().catch(() => {});
+    }
   }, BG_MONITOR_INTERVAL_MS);
-  _pollBackgroundStatus();
-  _checkServeReachability();
+  if (_canBackgroundPoll()) {
+    _pollBackgroundStatus();
+    _checkServeReachability();
+  }
 }
 
 function _stopBackgroundMonitor() {
@@ -3706,6 +3810,8 @@ async function _probeEndpointUntilOnline(epId, host, port) {
 }
 
 async function _pollBackgroundStatus() {
+  if (!_canBackgroundPoll() || _bgPollInFlight) return;
+  _bgPollInFlight = true;
   try {
     // Pull any tasks the server knows about that aren't in localStorage
     // yet (e.g. agent-spawned downloads/serves). Without this merge,
@@ -3749,6 +3855,34 @@ async function _pollBackgroundStatus() {
       const localTasks = _loadTasks();
       let changed = false;
       const completedDeps = [];
+      const localIds = new Set(localTasks.map(t => t.sessionId).filter(Boolean));
+      for (const live of tasks) {
+        const sid = live?.session_id;
+        if (!sid || localIds.has(sid) || _isTombstoned(sid)) continue;
+        const liveType = live.type || 'download';
+        const liveStatus = live.status === 'completed' ? 'done' : (live.status || 'running');
+        const name = live.model || sid;
+        const remoteHost = live.remote && live.remote !== 'local' ? live.remote : '';
+        localTasks.push(_redactTaskForStorage({
+          id: sid,
+          sessionId: sid,
+          name,
+          type: liveType,
+          status: liveStatus,
+          progress: live.progress || '',
+          output: live.output_tail || '',
+          ts: Date.now(),
+          payload: {
+            repo_id: name,
+            remote_host: remoteHost,
+            _cmd: live.cmd || '(adopted from live tmux status)',
+          },
+          remoteHost,
+          _adoptedExternally: true,
+        }));
+        localIds.add(sid);
+        changed = true;
+      }
       for (const task of localTasks) {
         const live = statusById.get(task.sessionId);
         if (!live) continue;
@@ -3788,7 +3922,9 @@ async function _pollBackgroundStatus() {
           const previous = String(task.output || '');
           const tail = String(live.output_tail || '');
           if (tail && !previous.endsWith(tail)) {
-            updates.output = `${previous ? `${previous}\n` : ''}${tail}`.slice(-5000);
+            updates.output = _isServeOutputPlaceholder(previous)
+              ? tail.slice(-5000)
+              : `${previous ? `${previous}\n` : ''}${tail}`.slice(-5000);
           }
         }
         if (live.diagnosis && !task._diagnosisDismissed) {
@@ -3941,6 +4077,8 @@ async function _pollBackgroundStatus() {
     }
   } catch (e) {
     // Silent fail
+  } finally {
+    _bgPollInFlight = false;
   }
 }
 
@@ -3969,17 +4107,15 @@ export function initRunning(shared) {
   _detectModelOptimizations = shared._detectModelOptimizations;
   _buildServeCmd = shared._buildServeCmd;
 
-  // App boot: pull authoritative state from server, then auto-start
-  // the background monitor unconditionally. Used to gate on "already
-  // has running tasks" but that meant when the agent (or anyone)
-  // added a task after boot, the UI never noticed. 10s poll of a
-  // small status endpoint is cheap and gives the agent + the UI a
-  // shared live picture.
+  // App boot: pull authoritative state from server, but don't start the
+  // running-task monitor unless there is real work to watch. Starting it
+  // unconditionally made a plain Cookbook open keep probing stale tmux/SSH
+  // sessions, which is expensive when a saved remote host is unreachable.
   (async () => {
     try {
       await _syncFromServer();
     } catch {}
-    _startBackgroundMonitor();
+    if (_hasLiveTasks()) _startBackgroundMonitor();
   })();
 }
 

static/js/cookbookServe.js

@@ -46,6 +46,30 @@ const SERVE_STATE_KEY = 'cookbook-serve-state';
 const SERVE_FAVORITES_KEY = 'cookbook-serve-favorite-models';
 
 let _cachedAllModels = [];
+const _CACHED_MODELS_SCAN_KEY = 'cookbook_cached_models_scan_v1';
+const _CACHED_MODELS_SCAN_TTL = 6 * 3600 * 1000;
+
+function _readCachedModelScan(sig) {
+  try {
+    const all = JSON.parse(localStorage.getItem(_CACHED_MODELS_SCAN_KEY) || '{}');
+    const entry = all[sig];
+    if (entry && Date.now() - (entry.ts || 0) < _CACHED_MODELS_SCAN_TTL) return entry.data || null;
+  } catch {}
+  return null;
+}
+
+function _writeCachedModelScan(sig, data) {
+  try {
+    const all = JSON.parse(localStorage.getItem(_CACHED_MODELS_SCAN_KEY) || '{}');
+    all[sig] = { ts: Date.now(), data };
+    const keys = Object.keys(all);
+    if (keys.length > 12) {
+      keys.sort((a, b) => (all[a].ts || 0) - (all[b].ts || 0));
+      for (const k of keys.slice(0, keys.length - 12)) delete all[k];
+    }
+    localStorage.setItem(_CACHED_MODELS_SCAN_KEY, JSON.stringify(all));
+  } catch {}
+}
 
 function _loadServeFavorites() {
   try {
@@ -490,12 +514,39 @@ function _estimateLlamaContextFit(model, fields, modelCtxMax, modelWeightsGb = 0
   }
   const raw = Math.floor(freeForKv / kvGbPerToken);
   const rounded = Math.max(1024, Math.floor(raw / 1024) * 1024);
-  const ctx = Math.min(modelMax, rounded);
+  let ctx = Math.min(modelMax, rounded);
+  let reasonSuffix = '';
+  if (isUnifiedMode) {
+    // Unified memory is not just "GPU math with a slightly bigger VRAM number".
+    // llama.cpp can spill into system RAM, so a conservative pure-VRAM KV
+    // formula makes confusing recommendations like "58G free unified" but the
+    // same context as GPU. Use a system-memory-style cap when there is real
+    // unified headroom, while keeping the GPU estimate as the minimum.
+    const unifiedCap = freeForKv >= 16
+      ? 131072
+      : (freeForKv >= 8 ? 65536 : 32768);
+    const unifiedCtx = Math.min(modelMax, unifiedCap);
+    if (unifiedCtx > ctx) {
+      ctx = unifiedCtx;
+      reasonSuffix = '; unified can spill into system RAM, slower than pure GPU';
+    }
+    const gpuUsableGb = Math.max(1, totalVramGb - Math.max(1.0, selectedCount * 0.6));
+    const gpuFreeForKv = gpuUsableGb - modelGb;
+    if (gpuFreeForKv > 0) {
+      const gpuRaw = Math.floor(gpuFreeForKv / kvGbPerToken);
+      const gpuRounded = Math.max(1024, Math.floor(gpuRaw / 1024) * 1024);
+      const gpuCtx = Math.min(modelMax, gpuRounded);
+      if (gpuCtx > ctx) {
+        ctx = gpuCtx;
+        reasonSuffix = '; at least the GPU estimate';
+      }
+    }
+  }
   return {
     ctx,
     modelGb,
     kvGbPerToken,
-    reason: `~${ctx.toLocaleString()} tokens fits llama.cpp KV (${freeForKv.toFixed(1)}G free ${isUnifiedMode ? 'unified' : 'VRAM'})`,
+    reason: `~${ctx.toLocaleString()} tokens fits llama.cpp KV (${freeForKv.toFixed(1)}G free ${isUnifiedMode ? 'unified' : 'VRAM'}${reasonSuffix})`,
   };
 }
 
@@ -1189,11 +1240,13 @@ function _rerenderCachedModels() {
       if (_replaceTaskId) {
         panelHtml += `<input type="hidden" class="hwfit-sf" data-field="_replaceTaskId" value="${esc(_replaceTaskId)}" />`;
       }
-      // Runtime-readiness note pinned at the top of the serve area so the
-      // user sees "vLLM ready on …" before scrolling into the configure
-      // form. Hidden until the readiness probe returns. The × button
-      // dismisses it for this panel only (re-shows on re-expand).
-      panelHtml += `<div class="hwfit-serve-runtime-note" style="display:none;font-size:11px;line-height:1.35;color:var(--fg-muted);margin:0 0 8px;padding:6px 28px 6px 10px;border-radius:5px;background:color-mix(in srgb, var(--fg) 4%, transparent);border:1px solid color-mix(in srgb, var(--border) 60%, transparent);position:relative;"><span class="hwfit-serve-runtime-text"></span><button type="button" class="hwfit-serve-runtime-close" title="Dismiss" aria-label="Dismiss" style="position:absolute;top:-8px;right:5px;background:none;border:0;color:inherit;cursor:pointer;padding:2px 4px;line-height:1;font-size:13px;opacity:0.6;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" aria-hidden="true"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg></button></div>`;
+      // Runtime-readiness note shares the top line with the preset controls
+      // so "vLLM ready on …" reads as panel status instead of a separate
+      // block pushing the form down. Hidden until the readiness probe returns.
+      panelHtml += `<div class="hwfit-serve-topline">`;
+      panelHtml += `<div class="hwfit-serve-runtime-note" style="display:none;font-size:11px;line-height:1.35;color:var(--fg-muted);margin:0;padding:6px 28px 6px 10px;border-radius:5px;background:color-mix(in srgb, var(--fg) 4%, transparent);border:1px solid color-mix(in srgb, var(--border) 60%, transparent);position:relative;"><span class="hwfit-serve-runtime-text"></span><button type="button" class="hwfit-serve-runtime-close" title="Dismiss" aria-label="Dismiss" style="position:absolute;top:-8px;right:5px;background:none;border:0;color:inherit;cursor:pointer;padding:2px 4px;line-height:1;font-size:13px;opacity:0.6;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" aria-hidden="true"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg></button></div>`;
+      panelHtml += `<div class="hwfit-serve-preset-row">${_slotsHtml}</div>`;
+      panelHtml += `</div>`;
       // Warn when serving a model whose download hasn't fully completed —
       // the user CAN still hit Launch (vLLM/llama-server will start, then
       // crash trying to read missing shards), but they should know.
@@ -1203,7 +1256,6 @@ function _rerenderCachedModels() {
           : `This model's download isn't complete yet (${esc(m.size || 'partial')}). The serve will start but is likely to crash on a missing shard. Wait for the download to finish, or relaunch after it's done.`;
         panelHtml += `<div class="hwfit-serve-warn" style="margin:0 0 8px;padding:6px 10px;border-radius:5px;font-size:11px;background:color-mix(in srgb, var(--color-warning, #f0ad4e) 14%, transparent);border:1px solid color-mix(in srgb, var(--color-warning, #f0ad4e) 40%, transparent);color:var(--color-warning, #f0ad4e);display:flex;gap:6px;align-items:flex-start;line-height:1.4;"><span aria-hidden="true">⚠</span><span>${_warnText}</span></div>`;
       }
-      panelHtml += `<div class="hwfit-serve-preset-row">${_slotsHtml}</div>`;
       panelHtml += `<div class="hwfit-serve-vision-warn" style="display:none;margin:0 0 8px;padding:6px 10px;border-radius:5px;font-size:11px;background:color-mix(in srgb, var(--color-warning, #f0ad4e) 14%, transparent);border:1px solid color-mix(in srgb, var(--color-warning, #f0ad4e) 40%, transparent);color:var(--color-warning, #f0ad4e);gap:6px;align-items:flex-start;line-height:1.4;"><span aria-hidden="true">⚠</span><span>Vision is enabled, but no mmproj GGUF projector was found in the cached model scan. Download an mmproj-*.gguf for this model, then refresh the cached model list before launching.</span></div>`;
       // Row 1: Engine + Server + Env
       panelHtml += `<div class="hwfit-serve-row">`;
@@ -1214,7 +1266,7 @@ function _rerenderCachedModels() {
       // stays as the source-of-truth so every existing change handler
       // (updateBackendVisibility, runtime readiness, command builder)
       // still fires via dispatchEvent('change') on selection.
-      panelHtml += `<label>${_l('Engine','Inference engine: vLLM, SGLang, llama.cpp, Ollama, or Diffusers')}<div class="hwfit-backend-picker" data-backend-picker style="position:relative;width:100%;"><select class="hwfit-sf hwfit-backend-source" data-field="backend" style="display:none;">${backendOpts}</select><button type="button" class="hwfit-backend-btn" data-backend-btn aria-haspopup="listbox" aria-expanded="false" style="display:flex;align-items:center;gap:6px;width:100%;height:28px;padding:0 8px;background:var(--bg);color:var(--fg);border:1px solid var(--border);border-radius:4px;font:inherit;font-size:11px;cursor:pointer;text-align:left;position:relative;top:-3px;"><span class="hwfit-backend-btn-icon" data-backend-icon-slot aria-hidden="true" style="display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;color:var(--accent, var(--red));flex-shrink:0;"></span><span class="hwfit-backend-btn-label" data-backend-label style="flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;"></span><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.6;flex-shrink:0;"><polyline points="6 9 12 15 18 9"/></svg></button><div class="hwfit-backend-menu" data-backend-menu role="listbox" hidden style="position:absolute;top:calc(100% + 4px);left:0;right:0;z-index:100;background:var(--panel, var(--bg));border:1px solid var(--border);border-radius:6px;box-shadow:0 6px 20px rgba(0,0,0,0.22);padding:4px;"></div></div></label>`;
+      panelHtml += `<label>${_l('Engine','Inference engine: vLLM, SGLang, llama.cpp, Ollama, or Diffusers')}<div class="hwfit-backend-picker" data-backend-picker style="position:relative;width:100%;"><select class="hwfit-sf hwfit-backend-source" data-field="backend" style="display:none;">${backendOpts}</select><button type="button" class="hwfit-backend-btn" data-backend-btn aria-haspopup="listbox" aria-expanded="false" style="display:flex;align-items:center;gap:6px;width:100%;height:32px;padding:0 8px;background:var(--bg);color:var(--fg);border:1px solid var(--border);border-radius:4px;font:inherit;font-size:11px;cursor:pointer;text-align:left;position:relative;top:-4px;"><span class="hwfit-backend-btn-icon" data-backend-icon-slot aria-hidden="true" style="display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;color:var(--accent, var(--red));flex-shrink:0;"></span><span class="hwfit-backend-btn-label" data-backend-label style="flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;"></span><svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" style="opacity:0.6;flex-shrink:0;"><polyline points="6 9 12 15 18 9"/></svg></button><div class="hwfit-backend-menu" data-backend-menu role="listbox" hidden style="position:absolute;top:calc(100% + 4px);left:0;right:0;z-index:100;background:var(--panel, var(--bg));border:1px solid var(--border);border-radius:6px;box-shadow:0 6px 20px rgba(0,0,0,0.22);padding:4px;"></div></div></label>`;
       panelHtml += `<input type="hidden" class="hwfit-sf" data-field="host" value="${esc(_es.remoteHost || '')}" />`;
       // Inference mode pill (llama.cpp only) — lives directly to the
       // RIGHT of Backend in Row 1 so the engine and the GPU/CPU choice
@@ -1238,9 +1290,9 @@ function _rerenderCachedModels() {
         const _savedUnified = !!sv('unified_mem', false);
         const _llamaModeRaw = sv('llama_mode', _llamaModeDefault);
         const _llamaMode = _savedUnified && _llamaModeRaw !== 'cpu' ? 'unified' : _llamaModeRaw;
-        panelHtml += `<label class="hwfit-backend-llamacpp">${_l('Inference','CPU = -ngl 0. GPU = -ngl 99. Unified = GPU offload plus GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 for unified-memory CUDA systems.')}<div class="mode-toggle mode-toggle-three${_llamaMode === 'gpu' ? ' mode-mid' : (_llamaMode === 'unified' ? ' mode-third' : '')}" data-llama-mode-toggle style="display:flex;width:100%;height:30px;position:relative;top:2px;"><button type="button" class="mode-toggle-btn${_llamaMode === 'cpu' ? ' active' : ''}" data-llama-mode="cpu" aria-pressed="${_llamaMode === 'cpu'}" style="flex:1;"><span style="position:relative;top:-7px;">CPU</span></button><button type="button" class="mode-toggle-btn${_llamaMode === 'gpu' ? ' active' : ''}" data-llama-mode="gpu" aria-pressed="${_llamaMode === 'gpu'}" style="flex:1;"><span style="position:relative;top:-7px;">GPU</span></button><button type="button" class="mode-toggle-btn${_llamaMode === 'unified' ? ' active' : ''}" data-llama-mode="unified" aria-pressed="${_llamaMode === 'unified'}" style="flex:1;"><span style="position:relative;top:-7px;">Unified</span></button></div><input type="hidden" class="hwfit-sf" data-field="llama_mode" value="${esc(_llamaMode)}" /><input type="hidden" class="hwfit-sf" data-field="unified_mem" value="${_llamaMode === 'unified' ? '1' : ''}" /></label>`;
+        panelHtml += `<label class="hwfit-backend-llamacpp">${_l('Inference','CPU = -ngl 0. GPU = -ngl 99. Unified = GPU offload plus GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 for unified-memory CUDA systems.')}<div class="mode-toggle mode-toggle-three${_llamaMode === 'gpu' ? ' mode-mid' : (_llamaMode === 'unified' ? ' mode-third' : '')}" data-llama-mode-toggle style="display:flex;width:100%;height:32px;position:relative;top:2px;"><button type="button" class="mode-toggle-btn${_llamaMode === 'cpu' ? ' active' : ''}" data-llama-mode="cpu" aria-pressed="${_llamaMode === 'cpu'}" style="flex:1;"><span style="position:relative;top:-7px;">CPU</span></button><button type="button" class="mode-toggle-btn${_llamaMode === 'gpu' ? ' active' : ''}" data-llama-mode="gpu" aria-pressed="${_llamaMode === 'gpu'}" style="flex:1;"><span style="position:relative;top:-7px;">GPU</span></button><button type="button" class="mode-toggle-btn${_llamaMode === 'unified' ? ' active' : ''}" data-llama-mode="unified" aria-pressed="${_llamaMode === 'unified'}" style="flex:1;"><span style="position:relative;top:-7px;">Unified</span></button></div><input type="hidden" class="hwfit-sf" data-field="llama_mode" value="${esc(_llamaMode)}" /><input type="hidden" class="hwfit-sf" data-field="unified_mem" value="${_llamaMode === 'unified' ? '1' : ''}" /></label>`;
       }
-      panelHtml += `<label>${_l('venv','Path to Python venv or conda env activate script')}<input type="text" class="hwfit-sf hwfit-sf-wide" data-field="venv" value="${esc(sv('venv', _es.envPath || _srvVenv || ''))}" placeholder="~/venv" /></label>`;
+      panelHtml += `<label>${_l('venv / conda','Path to a Python venv, or a Conda env name/path when the selected server uses Conda.')}<input type="text" class="hwfit-sf hwfit-sf-wide" data-field="venv" value="${esc(sv('venv', _es.envPath || _srvVenv || ''))}" placeholder="~/venv or conda-env" /></label>`;
       const defaultPort = defaultBackend === 'ollama' ? '11434' : _nextAvailablePort();
       panelHtml += `<label>${_l('Port','HTTP port for the API server')}<input type="text" class="hwfit-sf" data-field="port" value="${esc(sv('port', defaultPort))}" /></label>`;
       const _activeGpus = (defaultGpus || '').split(',').map(s => s.trim()).filter(Boolean);
@@ -1332,22 +1384,21 @@ function _rerenderCachedModels() {
           ['', 'None'],
           ['minimax_m3_cuda', 'CUDA native sampler'],
         ].map(([v, label]) => `<option value="${v}"${_envPresetVal === v ? ' selected' : ''}>${label}</option>`).join('');
-        panelHtml += `<label class="hwfit-backend-vllm" style="grid-column:1 / 2;">${_l('Env Preset','Adds known-good environment variables without typing them. CUDA native sampler adds VLLM_TARGET_DEVICE=cuda and disables FlashInfer sampler JIT; useful when system nvcc cannot compile the sampler for the GPU architecture.')}<select class="hwfit-sf" data-field="vllm_env_preset" style="height:32px;">${_envPresetOpts}</select></label>`;
+        panelHtml += `<label class="hwfit-backend-vllm" style="grid-column:1 / 2;">${_l('Env Preset','Adds known-good environment variables without typing them. CUDA native sampler adds VLLM_TARGET_DEVICE=cuda and disables FlashInfer sampler JIT; useful when system nvcc cannot compile the sampler for the GPU architecture.')}<select class="hwfit-sf" data-field="vllm_env_preset" style="height:32px;width:122px;">${_envPresetOpts}</select></label>`;
       }
       // Free-text env-vars field. Anything pasted here is prepended to the
       // launch command verbatim. Use for CUDACXX, PATH overrides, NCCL_*
       // tuning, or any other KEY=VALUE pair that doesn't have a dedicated
       // field. After the venv activate runs, $VIRTUAL_ENV / $PATH / etc. are
       // already exported so they expand correctly here.
-      // grid-column: 1 / -1 makes Env span every column of the Advanced
-      // row's CSS grid (the old flex:1 1 100% did nothing in a grid
-      // container — left an empty trailing column gap on wide modals).
-      panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang" style="grid-column:2 / -1;">${_l('Env','Extra KEY=VALUE env-var pairs prepended to the launch (space-separated). The Env Preset above covers the usual MiniMax M3 values; use this for additional overrides.')}<input type="text" class="hwfit-sf" data-field="extra_env" value="${esc(svm('extra_env', sv('extra_env','')))}" placeholder="NCCL_P2P_DISABLE=1" style="width:100%;" /></label>`;
+      // CSS places this beside vLLM's Env Preset, but lets it span the full
+      // row for SGLang where that preset field is hidden.
+      panelHtml += `<label class="hwfit-backend-vllm hwfit-backend-sglang hwfit-extra-env-label">${_l('Env','Extra KEY=VALUE env-var pairs prepended to the launch (space-separated). The Env Preset above covers the usual MiniMax M3 values; use this for additional overrides.')}<input type="text" class="hwfit-sf" data-field="extra_env" value="${esc(svm('extra_env', sv('extra_env','')))}" placeholder="NCCL_P2P_DISABLE=1" style="width:100%;" /></label>`;
       panelHtml += `</div>`;
       // Row 2b: Diffusers settings
       const diffDtypeOpts = ['bfloat16','float16','float32'].map(d => `<option value="${d}"${sv('diff_dtype','bfloat16')===d?' selected':''}>${d}</option>`).join('');
       const deviceMapOpts = ['balanced','auto','sequential'].map(d => `<option value="${d}"${sv('diff_device_map','balanced')===d?' selected':''}>${d}</option>`).join('');
-      panelHtml += `<div class="hwfit-serve-row hwfit-backend-diffusers">`;
+      panelHtml += `<div class="hwfit-serve-row hwfit-backend-diffusers hwfit-diff-settings-row">`;
       panelHtml += `<label>Dtype${_h('Precision. bfloat16 recommended for Flux, float16 for SD')} <select class="hwfit-sf" data-field="diff_dtype">${diffDtypeOpts}</select></label>`;
       panelHtml += `<label>Device Map${_h('How to place model on GPUs. balanced = split evenly')} <select class="hwfit-sf" data-field="diff_device_map">${deviceMapOpts}</select></label>`;
       panelHtml += `<label>Steps${_h('Default inference steps. More = better quality, slower')} <input type="text" class="hwfit-sf" data-field="diff_steps" value="${esc(sv('diff_steps', ''))}" placeholder="auto" /></label>`;
@@ -1407,21 +1458,21 @@ function _rerenderCachedModels() {
       const llamaSplitModeOpts = ['', 'layer', 'tensor', 'row', 'none'].map(d => `<option value="${d}"${sv('llama_split_mode','')===d?' selected':''}>${d||'default'}</option>`).join('');
 
       // Group 1 — GPU placement (GPU-only, hides in CPU mode)
-      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp cookbook-llama-gpu-only">`;
+      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp cookbook-llama-gpu-only hwfit-llama-placement-row">`;
       panelHtml += `<label>${_l('Split Mode','llama.cpp GPU placement. layer = default; tensor splits weights and KV across GPUs.')}<select class="hwfit-sf" data-field="llama_split_mode">${llamaSplitModeOpts}</select></label>`;
       panelHtml += `<label>${_l('Tensor Split','GPU proportions, e.g. 50,50 across two GPUs. Blank = auto.')}<input type="text" class="hwfit-sf" data-field="llama_tensor_split" value="${esc(sv('llama_tensor_split', ''))}" placeholder="auto" /></label>`;
       panelHtml += `<label>${_l('Main GPU','--main-gpu index inside the visible GPU set. Useful for split mode none/row.')}<input type="text" class="hwfit-sf" data-field="llama_main_gpu" value="${esc(sv('llama_main_gpu', ''))}" placeholder="auto" /></label>`;
       panelHtml += `</div>`;
 
       // Group 2 — Memory tuning (KV cache + MoE-on-CPU + Fit policy)
-      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp">`;
+      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp hwfit-llama-memory-row">`;
       panelHtml += `<label>${_l('KV Cache','cache-type-k/v: quantize the KV cache. q4_0 = smallest (more context), q8_0 = long-context, f16 = full.')}<select class="hwfit-sf" data-field="cache_type">${_kvOpts}</select></label>`;
       panelHtml += `<label class="cookbook-llama-gpu-only">${_l('CPU MoE','n-cpu-moe: number of MoE expert layers to run on CPU when the model is bigger than VRAM. 0 = all on GPU.')}<input type="text" class="hwfit-sf" data-field="n_cpu_moe" value="${esc(sv('n_cpu_moe',''))}" placeholder="0" /></label>`;
       panelHtml += `<label>${_l('Fit','llama.cpp --fit. Leave default unless you need explicit off/on behavior for a preset.')}<select class="hwfit-sf" data-field="llama_fit">${llamaFitOpts}</select></label>`;
       panelHtml += `</div>`;
 
       // Group 3 — Request batching (Batch / UBatch / Parallel)
-      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp">`;
+      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp hwfit-llama-batch-row">`;
       panelHtml += `<label>${_l('Batch','llama.cpp prompt batch size. Blank = default.')}<input type="text" class="hwfit-sf" data-field="llama_batch_size" value="${esc(sv('llama_batch_size', ''))}" placeholder="2048" /></label>`;
       panelHtml += `<label>${_l('UBatch','llama.cpp physical micro-batch size. Blank = default.')}<input type="text" class="hwfit-sf" data-field="llama_ubatch_size" value="${esc(sv('llama_ubatch_size', ''))}" placeholder="512" /></label>`;
       panelHtml += `<label>${_l('Parallel','llama.cpp parallel slots. Blank = default; 1 matches single-lane presets.')}<input type="text" class="hwfit-sf" data-field="llama_parallel" value="${esc(sv('llama_parallel', ''))}" placeholder="1" /></label>`;
@@ -1434,7 +1485,7 @@ function _rerenderCachedModels() {
       // Live VRAM / RAM-spillover monitor for the serve target's GPU. Polls
       // /api/cookbook/gpus while the panel is open so you can SEE whether the
       // config fits VRAM (fast) or spills to system RAM (slow). Populated after mount.
-      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp hwfit-vram-monitor" style="align-items:center;gap:8px;font-size:11px;">`;
+      panelHtml += `<div class="hwfit-serve-row hwfit-backend-llamacpp hwfit-vram-monitor hwfit-llama-monitor-row" style="align-items:center;gap:8px;font-size:11px;">`;
       panelHtml += `<span style="opacity:0.7;">GPU memory:</span>`;
       panelHtml += `<span class="hwfit-vram-readout" style="opacity:0.5;">checking…</span>`;
       panelHtml += `</div>`;
@@ -1443,20 +1494,20 @@ function _rerenderCachedModels() {
       // automatically in CPU mode. Order: perf-critical → safety → I/O →
       // niche. MTP Spec sits last because it owns its own numstep widget
       // and is the widest item.
-      panelHtml += `<div class="hwfit-serve-checks hwfit-backend-llamacpp">`;
+      panelHtml += `<div class="hwfit-serve-checks hwfit-backend-llamacpp hwfit-llama-checks-row">`;
       panelHtml += `<label class="hwfit-sf-cb cookbook-llama-gpu-only"><input type="checkbox" class="hwfit-sf" data-field="flash_attn"${sv('flash_attn',false)?' checked':''} /> Flash Attn${_h('--flash-attn on: faster attention + needed for quantized KV cache. Auto by default.')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb cookbook-llama-gpu-only"><input type="checkbox" class="hwfit-sf" data-field="llama_cpu_overflow"${sv('llama_cpu_overflow',false)?' checked':''} /> Allow CPU overflow${_h('OFF (default): cookbook blocks launches that would overflow GPU VRAM. ON: layers/KV cache that do not fit get pushed to CPU (slow).')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb cookbook-llama-gpu-only"><input type="checkbox" class="hwfit-sf" data-field="vision"${sv('vision',false)?' checked':''} /> Vision${_h('Serve with the vision encoder so the model can read images. Auto-finds an mmproj-*.gguf next to the model. Adds ~1 GB VRAM.')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="llama_no_mmap"${sv('llama_no_mmap',false)?' checked':''} /> No mmap${_h('Adds --no-mmap. Useful for some high-context/local-storage setups.')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="llama_no_warmup"${sv('llama_no_warmup',false)?' checked':''} /> Skip warmup${_h('Adds --no-warmup. Reduces startup memory spikes; llama.cpp defaults to warming up.')}</label>`;
-      panelHtml += `<label class="hwfit-sf-cb hwfit-spec-group"><input type="checkbox" class="hwfit-sf" data-field="llama_speculative_mtp"${sv('llama_speculative_mtp',false)?' checked':''} /> MTP Spec${_h('llama.cpp native MTP speculative decoding: --spec-type draft-mtp. Requires a GGUF with MTP heads.')} <span class="hwfit-numstep"><button type="button" class="hwfit-numstep-btn" data-step="-1" tabindex="-1" aria-label="Decrease">‹</button><input type="number" class="hwfit-sf hwfit-spec-tokens" data-field="llama_spec_tokens" value="${esc(sv('llama_spec_tokens', '3'))}" min="1" max="10" title="--spec-draft-n-max" /><button type="button" class="hwfit-numstep-btn" data-step="1" tabindex="-1" aria-label="Increase">›</button></span></label>`;
+      panelHtml += `<label class="hwfit-sf-cb hwfit-spec-group"><input type="checkbox" class="hwfit-sf" data-field="llama_speculative_mtp"${sv('llama_speculative_mtp',false)?' checked':''} /> MTP Spec${_h('llama.cpp native MTP speculative decoding: --spec-type draft-mtp. Requires a GGUF with MTP heads.')} <input type="number" class="hwfit-sf hwfit-spec-tokens hwfit-spec-tokens-bare" data-field="llama_spec_tokens" value="${esc(sv('llama_spec_tokens', '3'))}" min="1" max="10" title="--spec-draft-n-max" /></label>`;
       panelHtml += `</div>`;
       // Row 3b: Checkboxes (diffusers)
-      panelHtml += `<div class="hwfit-serve-checks hwfit-backend-diffusers">`;
+      panelHtml += `<div class="hwfit-serve-checks hwfit-backend-diffusers hwfit-diff-checks-row">`;
       panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="diff_offload"${sv('diff_offload',false)?' checked':''} /> CPU Offload${_h('Offload parts of model to CPU RAM to save VRAM. Slower but fits larger models')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="diff_attention_slicing"${sv('diff_attention_slicing',false)?' checked':''} /> Attention Slicing${_h('Slice attention computation to reduce peak VRAM. Slower')}</label>`;
       panelHtml += `<label class="hwfit-sf-cb"><input type="checkbox" class="hwfit-sf" data-field="diff_vae_slicing"${sv('diff_vae_slicing',false)?' checked':''} /> VAE Slicing${_h('Process VAE in slices. Reduces VRAM for high-res images')}</label>`;
-      panelHtml += `</div><div class="hwfit-serve-row hwfit-backend-diffusers">`;
+      panelHtml += `</div><div class="hwfit-serve-row hwfit-backend-diffusers hwfit-diff-harmonize-row">`;
       panelHtml += `<label>Harmonize GPU${_h('Separate GPU for img2img/harmonize. Leave empty to use same GPU')}<input type="text" class="hwfit-sf" data-field="diff_harmonize_gpu" value="${esc(sv('diff_harmonize_gpu', ''))}" placeholder="auto" style="width:50px;" /></label>`;
       panelHtml += `</div>`;
       // Model-specific optimizations. The checks row always renders for the
@@ -1821,7 +1872,7 @@ function _rerenderCachedModels() {
         vllm:   '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M3 4l7 16 7-16"/><path d="M14 4l4 9 3-9"/></svg>',
         sglang: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;mask:url(/static/icons/sglang-mark.png) center/contain no-repeat;"></span>',
         llamacpp: '<svg width="14" height="14" viewBox="0 0 600 600" fill="none" aria-hidden="true"><path d="M600 392L504.249 558L504.137 557.929C487.252 584.069 458.193 600 426.864 600H120L240 392H600Z" fill="currentColor"/><path d="M240 392H0L199.602 46.0254C216.032 17.5463 246.411 0 279.29 0H466.154L240 392Z" fill="currentColor"/></svg>',
-        ollama: '<img src="/static/icons/ollama-mark-crop.png" alt="" aria-hidden="true" width="14" height="14" style="display:block;width:14px;height:14px;object-fit:contain;" />',
+        ollama: '<span aria-hidden="true" style="display:block;width:14px;height:14px;background:currentColor;-webkit-mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;mask:url(/static/icons/ollama-mark-crop.png) center/contain no-repeat;"></span>',
         diffusers: '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v3M12 19v3M2 12h3M19 12h3M5 5l2 2M17 17l2 2M5 19l2-2M17 7l2-2"/></svg>',
       };
 
@@ -1902,6 +1953,7 @@ function _rerenderCachedModels() {
 
       function updateBackendVisibility() {
         const b = panel.querySelector('[data-field="backend"]')?.value || 'vllm';
+        panel.dataset.backendActive = b;
         panel.querySelectorAll('[class*="hwfit-backend-"]').forEach(el => {
           // Skip the entire backend-picker subtree — the picker's own
           // classes (`hwfit-backend-picker`, `-btn`, `-menu`, `-item`,
@@ -3274,7 +3326,7 @@ function _rerenderCachedModels() {
         // The venv field wins; otherwise fall back to the env configured for the
         // selected server in Settings, so the activation isn't silently dropped
         // when the field is left blank (the per-server venv wasn't being applied).
-        if (venvVal) { _envState.env = 'venv'; _envState.envPath = venvVal; }
+        if (venvVal) { _envState.env = (_srvEnv === 'conda' ? 'conda' : 'venv'); _envState.envPath = venvVal; }
         else if (_srvEnvPath) { _envState.env = (_srvEnv === 'conda' ? 'conda' : 'venv'); _envState.envPath = _srvEnvPath; }
         if (gpusVal) _envState.gpus = gpusVal;
         // Preflight: launching a GPU engine (llama.cpp / vLLM / SGLang)
@@ -3592,12 +3644,95 @@ export async function openServePanelForRepo(repo, fields) {
 
 // ── Fetch cached models from server ──
 
-export async function _fetchCachedModels() {
+function _renderCachedModelsData(list, data, host) {
+  // CHANGELOG: 'ready' already excludes partial downloads;
+  // show every complete model regardless of size/backend.
+  const ready = (data.models || []).filter(m => m.status === 'ready');
+
+  const downloading = (data.models || []).filter(m => m.status === 'downloading');
+  const allModels = [...ready, ...downloading];
+  _cachedAllModels = allModels;
+
+  if (!allModels.length) {
+    if (!host) {
+      list.innerHTML = '<div class="hwfit-loading" style="flex-direction:column;gap:6px;text-align:center;"><div>No cached models found</div><div style="font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;">Docker Local uses Odysseus’s cache in <code>data/huggingface</code>. Download a model here, or copy an existing host HuggingFace cache into that folder once.</div></div>';
+    } else {
+      list.innerHTML = '<div class="hwfit-loading" style="flex-direction:column;gap:8px;text-align:center;"><div>No cached models found</div><div style="font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;">No complete model folders were found on this server.</div><button type="button" class="hwfit-gpu-btn serve-empty-scan-btn" style="height:26px;padding:3px 10px;">Refresh</button></div>';
+      list.querySelector('.serve-empty-scan-btn')?.addEventListener('click', () => {
+        _fetchCachedModels(true);
+      });
+    }
+    const tagContainer = document.getElementById('serve-tags');
+    if (tagContainer) tagContainer.innerHTML = '';
+    return;
+  }
+
+  // Auto-detect type + family tags
+  const _tagMap = {};
+  const _familyMap = {};
+  const _families = [
+    [/qwen/i, 'qwen'], [/llama/i, 'llama'], [/mistral|mixtral/i, 'mistral'],
+    [/deepseek/i, 'deepseek'], [/gemma/i, 'gemma'], [/phi/i, 'phi'],
+    [/minimax/i, 'minimax'], [/glm/i, 'glm'], [/flux/i, 'flux'],
+    [/stable.?diffusion|sdxl/i, 'sd'], [/z-image/i, 'z-image'],
+    [/whisper/i, 'whisper'], [/command|cohere/i, 'cohere'],
+    [/yi-/i, 'yi'], [/intern/i, 'intern'], [/falcon/i, 'falcon'],
+  ];
+  for (const m of allModels) {
+    const n = (m.repo_id || '').toLowerCase();
+    let tag = 'other';
+    if (m.backend === 'ollama' || m.is_ollama) tag = 'llm';
+    else if (m.is_diffusion || /flux|sdxl|stable-diffusion|z-image|qwen-image|diffusion|dreamshar/i.test(n)) tag = 'image';
+    else if (/whisper|stt|asr/i.test(n)) tag = 'stt';
+    else if (/tts|cosyvoice|parler/i.test(n)) tag = 'tts';
+    else if (/embed|bge|minilm|e5-/i.test(n)) tag = 'embedding';
+    else if (/lora|adapter/i.test(n)) tag = 'lora';
+    else tag = 'llm';
+    m._tag = tag;
+    _tagMap[tag] = (_tagMap[tag] || 0) + 1;
+    m._family = '';
+    for (const [re, fam] of _families) {
+      if (re.test(n)) { m._family = fam; _familyMap[fam] = (_familyMap[fam] || 0) + 1; break; }
+    }
+    if ((m.backend === 'ollama' || m.is_ollama) && !m._family) {
+      m._family = 'ollama';
+      _familyMap.ollama = (_familyMap.ollama || 0) + 1;
+    }
+  }
+
+  // Render tag chips
+  const tagContainer = document.getElementById('serve-tags');
+  if (tagContainer) {
+    const tagOrder = ['llm', 'image', 'lora', 'embedding', 'tts', 'stt', 'other'];
+    let tagHtml = `<button class="memory-cat-chip active" data-serve-tag="">All (${allModels.length})</button>`;
+    for (const t of tagOrder) {
+      if (!_tagMap[t]) continue;
+      tagHtml += `<button class="memory-cat-chip" data-serve-tag="${t}">${t} (${_tagMap[t]})</button>`;
+    }
+    const sortedFamilies = Object.entries(_familyMap).sort((a, b) => b[1] - a[1]);
+    if (sortedFamilies.length) {
+      for (const [fam, count] of sortedFamilies) {
+        const logo = providerLogo(fam);
+        const logoHtml = logo ? `<span style="width:12px;height:12px;display:inline-flex;align-items:center;vertical-align:-2px;margin-right:2px;opacity:0.6;">${logo}</span>` : '';
+        tagHtml += `<button class="memory-cat-chip" data-serve-tag="fam:${fam}">${logoHtml}${fam} (${count})</button>`;
+      }
+    }
+    tagContainer.innerHTML = tagHtml;
+  }
+
+  _rerenderCachedModels();
+}
+
+export async function _fetchCachedModels(fresh = false, opts = {}) {
   const list = document.getElementById('hwfit-cached-list');
   if (!list) return;
+  const allowNetwork = fresh || opts.allowNetwork !== false;
 
   list.innerHTML = '';
-  const _dlWp = spinnerModule.createWhirlpool(18);
+  const _dlWp = spinnerModule.createWhirlpool(22);
+  _dlWp.element.classList.add('cookbook-section-loading-wp');
+  _dlWp.element.style.width = '22px';
+  _dlWp.element.style.height = '22px';
   const _dlWrap = document.createElement('div');
   _dlWrap.className = 'hwfit-loading';
   _dlWrap.style.cssText = 'flex-direction:column;gap:6px;';
@@ -3659,6 +3794,23 @@ export async function _fetchCachedModels() {
     if (host) { qp.set('host', host); const _sp4 = _getPort(host); if (_sp4) qp.set('ssh_port', _sp4); const _plat = _getPlatform(host); if (_plat) qp.set('platform', _plat); }
     if (modelDirs.length) qp.set('model_dir', modelDirs.join(','));
     const params = qp.toString() ? `?${qp}` : '';
+    const scanSig = params || 'local';
+    const cached = fresh ? null : _readCachedModelScan(scanSig);
+    if (cached) {
+      _dlWp.destroy();
+      _renderCachedModelsData(list, cached, host);
+      return;
+    }
+    if (!allowNetwork) {
+      _dlWp.destroy();
+      list.innerHTML = '<div class="hwfit-loading" style="flex-direction:column;gap:8px;text-align:center;"><div>No cached model scan yet</div><div style="font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;">Check this server\'s model cache.</div><button type="button" class="hwfit-gpu-btn serve-empty-scan-btn" style="height:26px;padding:3px 10px;">Scan</button></div>';
+      list.querySelector('.serve-empty-scan-btn')?.addEventListener('click', () => {
+        _fetchCachedModels(true);
+      });
+      const tagContainer = document.getElementById('serve-tags');
+      if (tagContainer) tagContainer.innerHTML = '';
+      return;
+    }
     const res = await fetch(`/api/model/cached${params}`);
     if (!res.ok) {
       const body = await res.text().catch(() => '');
@@ -3673,83 +3825,16 @@ export async function _fetchCachedModels() {
       throw new Error(`HTTP ${res.status} ${res.statusText}${msg ? `: ${msg}` : ''}`);
     }
     const data = await res.json();
+    if (data && data.error) throw new Error(data.error);
+    _writeCachedModelScan(scanSig, data);
     _dlWp.destroy();
-
-    // CHANGELOG: 'ready' already excludes partial downloads; 
-    // show every complete model regardless of size/backend.
-    const ready = data.models.filter(m => m.status === 'ready');
-
-    const downloading = data.models.filter(m => m.status === 'downloading');
-    const allModels = [...ready, ...downloading];
-    _cachedAllModels = allModels;
-
-    if (!allModels.length) {
-      if (!host) {
-        list.innerHTML = '<div class="hwfit-loading" style="flex-direction:column;gap:6px;text-align:center;"><div>No cached models found</div><div style="font-size:11px;opacity:0.55;max-width:420px;line-height:1.4;">Docker Local uses Odysseus’s cache in <code>data/huggingface</code>. Download a model here, or copy an existing host HuggingFace cache into that folder once.</div></div>';
-      } else {
-        list.innerHTML = '<div class="hwfit-loading">No cached models found</div>';
-      }
-      document.getElementById('serve-tags').innerHTML = '';
-      return;
-    }
-
-    // Auto-detect type + family tags
-    const _tagMap = {};
-    const _familyMap = {};
-    const _families = [
-      [/qwen/i, 'qwen'], [/llama/i, 'llama'], [/mistral|mixtral/i, 'mistral'],
-      [/deepseek/i, 'deepseek'], [/gemma/i, 'gemma'], [/phi/i, 'phi'],
-      [/minimax/i, 'minimax'], [/glm/i, 'glm'], [/flux/i, 'flux'],
-      [/stable.?diffusion|sdxl/i, 'sd'], [/z-image/i, 'z-image'],
-      [/whisper/i, 'whisper'], [/command|cohere/i, 'cohere'],
-      [/yi-/i, 'yi'], [/intern/i, 'intern'], [/falcon/i, 'falcon'],
-    ];
-    for (const m of allModels) {
-      const n = (m.repo_id || '').toLowerCase();
-      let tag = 'other';
-      if (m.backend === 'ollama' || m.is_ollama) tag = 'llm';
-      else if (m.is_diffusion || /flux|sdxl|stable-diffusion|z-image|qwen-image|diffusion|dreamshar/i.test(n)) tag = 'image';
-      else if (/whisper|stt|asr/i.test(n)) tag = 'stt';
-      else if (/tts|cosyvoice|parler/i.test(n)) tag = 'tts';
-      else if (/embed|bge|minilm|e5-/i.test(n)) tag = 'embedding';
-      else if (/lora|adapter/i.test(n)) tag = 'lora';
-      else tag = 'llm';
-      m._tag = tag;
-      _tagMap[tag] = (_tagMap[tag] || 0) + 1;
-      m._family = '';
-      for (const [re, fam] of _families) {
-        if (re.test(n)) { m._family = fam; _familyMap[fam] = (_familyMap[fam] || 0) + 1; break; }
-      }
-      if ((m.backend === 'ollama' || m.is_ollama) && !m._family) {
-        m._family = 'ollama';
-        _familyMap.ollama = (_familyMap.ollama || 0) + 1;
-      }
-    }
-
-    // Render tag chips
-    const tagContainer = document.getElementById('serve-tags');
-    if (tagContainer) {
-      const tagOrder = ['llm', 'image', 'lora', 'embedding', 'tts', 'stt', 'other'];
-      let tagHtml = `<button class="memory-cat-chip active" data-serve-tag="">All (${allModels.length})</button>`;
-      for (const t of tagOrder) {
-        if (!_tagMap[t]) continue;
-        tagHtml += `<button class="memory-cat-chip" data-serve-tag="${t}">${t} (${_tagMap[t]})</button>`;
-      }
-      const sortedFamilies = Object.entries(_familyMap).sort((a, b) => b[1] - a[1]);
-      if (sortedFamilies.length) {
-        for (const [fam, count] of sortedFamilies) {
-          const logo = providerLogo(fam);
-          const logoHtml = logo ? `<span style="width:12px;height:12px;display:inline-flex;align-items:center;vertical-align:-2px;margin-right:2px;opacity:0.6;">${logo}</span>` : '';
-          tagHtml += `<button class="memory-cat-chip" data-serve-tag="fam:${fam}">${logoHtml}${fam} (${count})</button>`;
-        }
-      }
-      tagContainer.innerHTML = tagHtml;
-    }
-
-    _rerenderCachedModels();
+    _renderCachedModelsData(list, data, host);
   } catch (e) {
     _dlWp.destroy();
-    list.innerHTML = `<div class="hwfit-loading">Failed: ${esc(e.message)}</div>`;
+    list.innerHTML = `<div class="hwfit-loading" style="flex-direction:column;gap:8px;text-align:center;"><div style="color:var(--red);font-weight:600;">Cached model scan failed</div><div style="font-size:11px;opacity:0.65;max-width:420px;line-height:1.4;">${esc(e.message)}</div><button type="button" class="hwfit-gpu-btn serve-empty-scan-btn" style="height:26px;padding:3px 10px;">Retry</button></div>`;
+    list.querySelector('.serve-empty-scan-btn')?.addEventListener('click', () => {
+      _fetchCachedModels(true);
+    });
   }
 }
 

static/js/editor/harmonize-masks.js

@@ -32,6 +32,7 @@
  * @returns {HTMLCanvasElement|null}
  */
 export function layerUnionAlpha(w, h, layers) {
+  if (!Array.isArray(layers)) return null;
   const visible = layers.filter(l => l.visible);
   if (visible.length < 2) return null;
   const bgId = visible[0].id;

static/js/editor/snap.js

@@ -23,8 +23,10 @@
  * @returns {{x: number, y: number, guides: Array}}
  */
 export function computeSnap(layer, nx, ny, ctx) {
-  const SNAP_PX = 6 / Math.max(ctx.zoom, 0.0001);
-  const cw = ctx.canvasW, ch = ctx.canvasH;
+  if (!layer || !layer.canvas || !ctx) return { x: nx, y: ny, guides: [] };
+  const zoom = Number.isFinite(Number(ctx.zoom)) ? Number(ctx.zoom) : 1;
+  const SNAP_PX = 6 / Math.max(zoom, 0.0001);
+  const cw = Number(ctx.canvasW) || 0, ch = Number(ctx.canvasH) || 0;
   const w = layer.canvas.width, h = layer.canvas.height;
 
   const vTargets = [

static/js/emailInbox.js

@@ -5,16 +5,15 @@
 
 import spinnerModule from './spinner.js';
 import sessionModule from './sessions.js';
-import { initEmailLibrary, openEmailLibrary, closeEmailLibrary, isOpen as isLibOpen, prewarmEmailLibrary } from './emailLibrary.js';
+import { initEmailLibrary, openEmailLibrary, closeEmailLibrary, isOpen as isLibOpen } from './emailLibrary.js';
 import * as Modals from './modalManager.js';
 import { applyEdgeDock } from './modalSnap.js';
 import { buildReplyAllCc } from './emailLibrary/replyRecipients.js';
+import { emailApiUrl, emailAccountQuery } from './emailShared.js';
 import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
 
 const API_BASE = window.location.origin;
-const _acct = () => window.__odysseusActiveEmailAccount
-  ? `&account_id=${encodeURIComponent(window.__odysseusActiveEmailAccount)}`
-  : '';
+const _acct = () => emailAccountQuery('&');
 
 const _emailSetupHint = () => '<div style="margin-top:6px;opacity:0.72;font-size:11px;">Setup: <span style="color:var(--accent,var(--red));">Settings &rsaquo; Integrations</span></div>';
 
@@ -28,6 +27,59 @@ const _starFilledIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="c
 const _bellIcon = '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 8A6 6 0 0 0 6 8c0 7-3 9-3 9h18s-3-2-3-9"/><path d="M13.73 21a2 2 0 0 1-3.46 0"/></svg>';
 const _icon = (svg) => `<span class="dropdown-icon">${svg}</span>`;
 const _replySeparator = '---------- Previous message ----------';
+const _DONE_RESPONSE_TAGS = new Set(['urgent', 'reply-soon', 'action-needed']);
+
+function _openCalendarEventFromEmail(uid) {
+  const target = String(uid || '').trim();
+  if (!target) return;
+  import('./calendar.js').then(mod => {
+    const open = mod.openCalendarTo || (mod.default && mod.default.openCalendarTo);
+    if (open) open(target);
+  }).catch(() => {});
+}
+
+function _openEmailTagFilter(tag) {
+  const normalized = String(tag || '').trim().toLowerCase().replace(/_/g, '-');
+  if (!normalized || normalized === 'calendar') return;
+  try { openEmailLibrary(); } catch (_) {}
+  setTimeout(() => {
+    document.dispatchEvent(new CustomEvent('odysseus:email-filter-tag', { detail: { tag: normalized } }));
+  }, 0);
+}
+
+function _emailTagPillHtml(tag, em) {
+  const normalized = String(tag || '').trim().toLowerCase().replace(/_/g, '-');
+  if (!normalized) return '';
+  const eventUid = normalized === 'calendar' && Array.isArray(em?.calendar_event_uids)
+    ? String(em.calendar_event_uids[0] || '').trim()
+    : '';
+  if (normalized === 'calendar') {
+    if (!eventUid) return '';
+    return `<button type="button" class="email-tag email-tag-${_esc(normalized)} email-tag-clickable" data-calendar-event-uid="${_esc(eventUid)}" title="Open calendar event">${_esc(normalized)}</button>`;
+  }
+  return `<button type="button" class="email-tag email-tag-${_esc(normalized)} email-tag-clickable" data-email-filter-tag="${_esc(normalized)}" title="Show ${_esc(normalized)} emails">${_esc(normalized)}</button>`;
+}
+
+function _emailTagGroupHtml(tags, em) {
+  const visible = (Array.isArray(tags) ? tags : [])
+    .map(t => _emailTagPillHtml(t, em))
+    .filter(Boolean);
+  if (!visible.length) return '';
+  if (visible.length === 1) return `<span class="email-tags">${visible[0]}</span>`;
+  const extra = visible.slice(1).map(html => `<span class="email-tag-extra">${html}</span>`).join('');
+  return `<span class="email-tags email-tags-collapsed">${visible[0]}${extra}<button type="button" class="email-tags-more" data-email-tags-more aria-expanded="false" title="Show all tags">+${visible.length - 1}<svg width="9" height="9" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="6 9 12 15 18 9"></polyline></svg></button></span>`;
+}
+
+function _visibleEmailTagsForRender(em) {
+  const tags = Array.isArray(em?.tags) ? em.tags : [];
+  if (!em?.is_answered) return tags;
+  return tags.filter(t => !_DONE_RESPONSE_TAGS.has(String(t || '').trim().toLowerCase().replace(/_/g, '-')));
+}
+
+function _clearDoneResponseTagsLocal(em) {
+  if (!em || !Array.isArray(em.tags)) return;
+  em.tags = em.tags.filter(t => !_DONE_RESPONSE_TAGS.has(String(t || '').trim().toLowerCase().replace(/_/g, '-')));
+}
 
 function _cleanAiReplyText(text) {
   if (!text) return '';
@@ -70,9 +122,14 @@ window.addEventListener('email-answered', (e) => {
   const uid = e.detail && e.detail.uid;
   if (uid == null) return;
   const em = _emails.find(x => String(x.uid) === String(uid));
-  if (em) { em.is_answered = true; em.is_read = true; }
+  if (em) {
+    em.is_answered = true;
+    em.is_read = true;
+    _clearDoneResponseTagsLocal(em);
+  }
   document.querySelectorAll('.email-item[data-uid="' + CSS.escape(String(uid)) + '"]').forEach(item => {
     item.classList.remove('email-unread');
+    item.querySelectorAll('.email-tag-urgent, .email-tag-reply-soon, .email-tag-action-needed').forEach(n => n.remove());
     const check = item.querySelector('.email-done-check');
     if (check) check.classList.add('active');
     // Auto-mark from sending a reply — flash the row so the user sees the
@@ -88,10 +145,15 @@ let _docModule = null;
 let _listSpinner = null;
 let _senderFilter = null;       // email address (lowercased) to filter by, or null
 let _senderFilterLabel = null;  // display label for the active filter chip
+let _showEmailTags = localStorage.getItem('odysseus.email.showTags') !== '0';
 
 export function init(documentModule) {
   _docModule = documentModule;
   _bindEvents();
+  document.addEventListener('odysseus:email-tags-toggle', (e) => {
+    _showEmailTags = e.detail?.show !== false;
+    _renderList();
+  });
   // Init the library popup with a callback to open emails
   initEmailLibrary({
     documentModule,
@@ -137,6 +199,23 @@ export async function openReplyDraft(uid, folder = 'INBOX', mode = 'reply', pref
   }
 }
 
+function _bringEmailReplyDraftToFrontOnMobile() {
+  if (window.innerWidth > 768) return;
+  document.body.classList.remove('email-front', 'email-doc-split-active');
+  document.documentElement.style.removeProperty('--email-doc-split-left-x');
+  document.documentElement.style.removeProperty('--email-doc-split-email-w');
+  document.documentElement.style.removeProperty('--email-doc-split-right-x');
+  // Keep the email sheet visible behind the reply document on mobile. The
+  // document panel sits above it via the normal doc-view z-index rules, and
+  // swiping the document down minimizes it to a chip to reveal the email.
+  document.querySelectorAll('#email-lib-modal, .modal[id^="email-reader-"]').forEach(modal => {
+    modal.classList.remove('email-snap-left', 'modal-left-docked', 'modal-right-docked');
+    modal.style.removeProperty('z-index');
+  });
+  const docPane = document.getElementById('doc-editor-pane');
+  if (docPane) docPane.style.setProperty('z-index', '10010', 'important');
+}
+
 // When the document editor pane opens (body.doc-view turns on), make sure the
 // email modal is on the LEFT — even if it was previously docked RIGHT or
 // floating — so the email and the doc always end up side-by-side. The actual
@@ -195,10 +274,11 @@ function _bindEvents() {
     });
   }
 
-  // Initial unread count check, refresh every 60s
-  _refreshUnreadCount();
+  // Delay the lightweight unread badge check so opening Odysseus doesn't
+  // compete with the initial chat/session paint. The full email list now loads
+  // only when the inbox is actually opened.
+  setTimeout(_refreshUnreadCount, 8000);
   setInterval(_refreshUnreadCount, 60000);
-  prewarmEmailLibrary({ delay: 3000 });
 
   // Deep-link: #email=<folder>:<uid> opens the library and expands that card
   _maybeOpenFromHash();
@@ -231,24 +311,24 @@ async function _refreshUnreadCount() {
   const dot = document.getElementById('email-unread-dot');
   if (dot && !dot._stickyState) dot.style.display = 'none';
   try {
-    // Parallel: unread list + urgency state.
-    const [listRes, urgRes] = await Promise.all([
-      fetch(`${API_BASE}/api/email/list?folder=INBOX&limit=50&filter=unread${_acct()}`),
+    // Parallel: cheap unread state + urgency state.
+    const [stateRes, urgRes] = await Promise.all([
+      fetch(emailApiUrl('/api/email/unread-state', { folder: 'INBOX' })),
       fetch(`${API_BASE}/api/email/urgency-state`, { credentials: 'same-origin' }).catch(() => null),
     ]);
-    if (!listRes || !listRes.ok) return;
-    const data = await listRes.json();
+    if (!stateRes || !stateRes.ok) return;
+    const data = await stateRes.json();
     if (!dot) return;
 
-    const emails = data.emails || [];
-    if (emails.length === 0) {
+    const unreadCount = Number(data.unread_count || 0);
+    if (unreadCount <= 0) {
       dot.style.display = 'none';
       return;
     }
 
     // Compare highest unread UID to the last-seen threshold in localStorage
     const lastSeen = parseInt(localStorage.getItem('odysseus-email-last-seen-uid') || '0', 10);
-    const maxUid = Math.max(...emails.map(e => parseInt(e.uid, 10) || 0));
+    const maxUid = parseInt(data.max_uid || '0', 10) || 0;
 
     // Only show dot if there's a new email above the threshold
     dot.style.display = maxUid > lastSeen ? '' : 'none';
@@ -276,12 +356,11 @@ export function markInboxAsSeen() {
   // Called when the user opens the inbox popup — clears the notif dot
   try {
     // Find current max UID so subsequent arrivals trigger the dot
-    fetch(`${API_BASE}/api/email/list?folder=INBOX&limit=1${_acct()}`)
+    fetch(emailApiUrl('/api/email/unread-state', { folder: 'INBOX' }))
       .then(r => r.json())
       .then(data => {
-        const emails = data.emails || [];
-        if (emails.length > 0) {
-          const maxUid = Math.max(...emails.map(e => parseInt(e.uid, 10) || 0));
+        const maxUid = parseInt(data.max_uid || '0', 10) || 0;
+        if (maxUid > 0) {
           localStorage.setItem('odysseus-email-last-seen-uid', String(maxUid));
         }
         const dot = document.getElementById('email-unread-dot');
@@ -309,7 +388,7 @@ export async function loadEmails(append = false) {
 
   try {
     const fromQS = _senderFilter ? `&from=${encodeURIComponent(_senderFilter)}` : '';
-    const res = await fetch(`${API_BASE}/api/email/list?folder=${encodeURIComponent(_currentFolder)}&limit=50&offset=${_offset}${fromQS}${_acct()}&_=${Date.now()}`);
+    const res = await fetch(`${API_BASE}/api/email/list?folder=${encodeURIComponent(_currentFolder)}&limit=50&offset=${_offset}${fromQS}${_acct()}`);
     const data = await res.json();
     if (data.error) throw new Error(data.error);
 
@@ -339,7 +418,8 @@ export async function loadEmails(append = false) {
 
 async function loadFolders() {
   try {
-    const res = await fetch(`${API_BASE}/api/email/folders?_=1${_acct()}`);
+    const accountQS = _acct().replace(/^&/, '');
+    const res = await fetch(`${API_BASE}/api/email/folders${accountQS ? `?${accountQS}` : ''}`);
     const data = await res.json();
     const select = document.getElementById('email-folder-select');
     if (!select || !data.folders) return;
@@ -512,12 +592,10 @@ function _createEmailItem(em) {
     ? `<span class="email-unread-dot-inline" title="${_esc(_unreadTitle)}" style="display:inline-flex;align-items:center;flex-shrink:0;margin-left:4px;color:${_unreadColor}"><svg width="8" height="8" viewBox="0 0 24 24" fill="currentColor"><circle cx="12" cy="12" r="6"/></svg></span>`
     : '';
 
-  const tags = Array.isArray(em.tags) ? em.tags : [];
-  const tagPills = tags.length
-    ? `<span class="email-tags">${tags.map(t => `<span class="email-tag email-tag-${_esc(t)}">${_esc(t)}</span>`).join('')}</span>`
-    : '';
+  const tags = _showEmailTags ? _visibleEmailTagsForRender(em) : [];
+  const tagPills = _emailTagGroupHtml(tags, em);
 
-  const spamTag = em.is_spam_verdict
+  const spamTag = _showEmailTags && em.is_spam_verdict
     ? `<span class="email-tag email-tag-spam" title="AI flagged as spam — click ✓ to unflag">spam <button class="email-spam-unflag" data-uid="${em.uid}" title="Not spam">\u2713</button></span>`
     : '';
 
@@ -535,6 +613,30 @@ function _createEmailItem(em) {
 
   // Click sender name → filter list to that sender
   const senderEl = item.querySelector('.email-sender-clickable');
+  item.querySelectorAll('[data-calendar-event-uid]').forEach(btn => {
+    btn.addEventListener('click', (ev) => {
+      ev.preventDefault();
+      ev.stopPropagation();
+      _openCalendarEventFromEmail(btn.dataset.calendarEventUid);
+    });
+  });
+  item.querySelectorAll('[data-email-filter-tag]').forEach(btn => {
+    btn.addEventListener('click', (ev) => {
+      ev.preventDefault();
+      ev.stopPropagation();
+      _openEmailTagFilter(btn.dataset.emailFilterTag);
+    });
+  });
+  item.querySelectorAll('[data-email-tags-more]').forEach(btn => {
+    btn.addEventListener('click', (ev) => {
+      ev.preventDefault();
+      ev.stopPropagation();
+      const wrap = btn.closest('.email-tags');
+      if (!wrap) return;
+      const expanded = wrap.classList.toggle('email-tags-expanded');
+      btn.setAttribute('aria-expanded', expanded ? 'true' : 'false');
+    });
+  });
   if (senderEl) {
     senderEl.addEventListener('click', (e) => {
       e.stopPropagation();
@@ -659,7 +761,8 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
   try {
     let data = preloadedData;
     if (!data) {
-      const res = await fetch(`${API_BASE}/api/email/read/${em.uid}?folder=${encodeURIComponent(_currentFolder)}${_acct()}`);
+      const fullQS = mode === 'forward' ? '&full=1' : '';
+      const res = await fetch(`${API_BASE}/api/email/read/${em.uid}?folder=${encodeURIComponent(_currentFolder)}${_acct()}${fullQS}`);
       data = await res.json();
     }
     if (data.error) {
@@ -702,7 +805,10 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
           if (result.success && result.reply) {
             aiSuggestedBody = _cleanAiReplyText(result.reply);
           } else {
-            const _msg = result.error || 'AI reply could not be generated';
+            const _rawMsg = result.error || 'AI reply could not be generated';
+            const _msg = /empty response/i.test(_rawMsg)
+              ? 'AI returned empty response.'
+              : _rawMsg;
             console.error('AI reply generation failed:', _msg);
             import('./ui.js').then(m => m.showError && m.showError('AI reply failed: ' + _msg)).catch(() => {});
             return;
@@ -753,6 +859,7 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
     if (data.attachments && data.attachments.length > 0) {
       const attStr = data.attachments.map(a => `${a.index}:${a.filename}:${a.size}`).join('|');
       content += `\nX-Attachments: ${attStr}`;
+      if (mode === 'forward') content += `\nX-Forward-Attachments: 1`;
     }
     content += '\n---\n';
 
@@ -811,12 +918,10 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
     }
 
     if (_docModule) {
-      // Only reuse an existing doc tab if the user really just wants to "view"
-      // the email again. For reply/reply-all/forward/ai-reply, always create
-      // a fresh draft — otherwise a previously-emptied doc (sent reply, AI
-      // reply that came back blank, etc.) keeps coming back instead of a
-      // proper pre-filled reply.
-      const reuseExisting = (mode === 'view' || mode === 'open');
+      // Agent-provided reply text should land in the email draft the user
+      // already has open. Otherwise mobile users see the source email while the
+      // agent silently creates a second draft elsewhere.
+      const reuseExisting = (mode === 'view' || mode === 'open' || (!!aiSuggestedBody && mode !== 'forward'));
       const existingDocId = (reuseExisting && _docModule.findEmailDocId)
         ? _docModule.findEmailDocId(em.uid, _currentFolder)
         : null;
@@ -824,6 +929,10 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
         if (!_docModule.isPanelOpen()) _docModule.openPanel();
         await new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)));
         await _docModule.loadDocument(existingDocId);
+        if (aiSuggestedBody && typeof _docModule.replaceEmailReplyBody === 'function') {
+          await _docModule.replaceEmailReplyBody(existingDocId, aiSuggestedBody);
+          _bringEmailReplyDraftToFrontOnMobile();
+        }
       } else {
         // If the user already has a chat session open, reuse it instead of
         // spawning a new one. They asked for this explicitly — opening reply
@@ -896,6 +1005,7 @@ async function _openEmail(em, itemEl, preloadedData = null, mode = 'reply', note
           } else {
             await _docModule.loadDocument(doc.id);
           }
+          _bringEmailReplyDraftToFrontOnMobile();
         }
       }
     }
@@ -1079,24 +1189,56 @@ async function _deleteEmail(em) {
   const { styledConfirm } = await import('./ui.js');
   const ok = await styledConfirm(`Delete "${subject}"?`, { confirmText: 'Delete', cancelText: 'Cancel', danger: true });
   if (!ok) return;
+  const row = document.querySelector(`.email-item[data-uid="${CSS.escape(String(em.uid))}"]`);
+  const busy = _showEmailDeleteOverlay(row);
+  await busy?.ready;
   try {
     await fetch(`${API_BASE}/api/email/delete/${em.uid}?folder=${encodeURIComponent(_currentFolder)}${_acct()}`, { method: 'DELETE' });
+    busy?.remove?.();
     _emails = _emails.filter(e => e.uid !== em.uid);
     _renderList();
   } catch (e) {
+    busy?.remove?.();
     console.error('Failed to delete:', e);
   }
 }
 
+function _showEmailDeleteOverlay(target) {
+  if (!target) return null;
+  const wp = spinnerModule.createWhirlpool(16);
+  const overlay = document.createElement('div');
+  overlay.className = 'email-delete-overlay';
+  overlay.appendChild(wp.element);
+  const prevPos = target.style.position;
+  const prevPointerEvents = target.style.pointerEvents;
+  if (getComputedStyle(target).position === 'static') target.style.position = 'relative';
+  target.style.pointerEvents = 'none';
+  target.classList.add('email-delete-busy');
+  target.appendChild(overlay);
+  const ready = new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
+  return {
+    ready,
+    remove() {
+      try { wp.destroy?.(); } catch (_) {}
+      overlay.remove();
+      target.classList.remove('email-delete-busy');
+      target.style.pointerEvents = prevPointerEvents;
+      target.style.position = prevPos;
+    }
+  };
+}
+
 async function _toggleDone(em, itemEl) {
   const newState = !em.is_answered;
   em.is_answered = newState;
   if (newState) em.is_read = true; // mark-done implies mark-read
   if (itemEl) {
     if (newState) {
+      _clearDoneResponseTagsLocal(em);
       itemEl.classList.remove('email-unread');
       // Also drop any inline unread indicator dots the renderer may have added
       itemEl.querySelectorAll('.email-unread-dot, [data-unread-dot]').forEach(n => n.remove());
+      itemEl.querySelectorAll('.email-tag-urgent, .email-tag-reply-soon, .email-tag-action-needed').forEach(n => n.remove());
     }
     const check = itemEl.querySelector('.email-done-check');
     if (check) check.classList.toggle('active', newState);

static/js/emailLibrary/state.js

@@ -25,6 +25,7 @@ export const state = {
   _libFilter: 'all',             // all, unread, unanswered
   _libSort: 'recent',            // recent, unread, favorites
   _libHasAttachments: false,
+  _libShowTags: localStorage.getItem('odysseus.email.showTags') !== '0',
   _libLoading: false,
   _docModule: null,
   _onEmailClick: null,

static/js/emailShared.js

@@ -0,0 +1,19 @@
+const API_BASE = window.location.origin;
+
+export function emailAccountQuery(prefix = '&') {
+  const accountId = window.__odysseusActiveEmailAccount || '';
+  if (!accountId) return '';
+  const lead = prefix === '?' ? '?' : '&';
+  return `${lead}account_id=${encodeURIComponent(accountId)}`;
+}
+
+export function emailApiUrl(path, params = {}) {
+  const url = new URL(`${API_BASE}${path}`);
+  const accountId = window.__odysseusActiveEmailAccount || '';
+  if (accountId) url.searchParams.set('account_id', accountId);
+  Object.entries(params || {}).forEach(([key, value]) => {
+    if (value === undefined || value === null || value === '') return;
+    url.searchParams.set(key, String(value));
+  });
+  return url.toString();
+}

static/js/fileHandler.js

@@ -15,6 +15,9 @@ let uploaded = [];
 let _lastUploadedMeta = [];
 let API_BASE = '';
 let _uploadSpinners = [];
+let _uploadAbortCtrl = null;
+let _uploading = false;
+let _lastUploadCancelled = false;
 const _previewUrls = new WeakMap();
 
 const MAX_FILES = 10;
@@ -130,6 +133,7 @@ function _createChip(f, idx) {
  * Remove a pending file by index
  */
 export function removePending(idx) {
+  if (_uploading) cancelUpload();
   _revokePreviewUrl(pendingFiles[idx]);
   pendingFiles.splice(idx, 1);
   renderAttachStrip();
@@ -138,8 +142,9 @@ export function removePending(idx) {
 /**
  * Upload all pending files to server
  */
-export async function uploadPending() {
+export async function uploadPending(opts = {}) {
   if (pendingFiles.length === 0) return [];
+  _lastUploadCancelled = false;
 
   // The message bubble is shown immediately, but the upload can take a moment —
   // dim the chips and overlay a whirlpool so it's clear the files are still
@@ -164,11 +169,20 @@ export async function uploadPending() {
 
   const fd = new FormData();
   pendingFiles.forEach(f => fd.append('files', f, f.name || 'paste.png'));
+  if (opts.sessionId) fd.append('session_id', opts.sessionId);
+  _uploadAbortCtrl = new AbortController();
+  _uploading = true;
+  const timeoutId = setTimeout(() => {
+    if (_uploadAbortCtrl && !_uploadAbortCtrl.signal.aborted) {
+      try { _uploadAbortCtrl.abort(); } catch (_) {}
+    }
+  }, 120000);
 
   try {
     const res = await fetch(`${API_BASE}/api/upload`, {
       method: 'POST',
-      body: fd
+      body: fd,
+      signal: _uploadAbortCtrl.signal,
     });
     if (!res.ok) {
       // Surface the failure instead of swallowing it. Previously a non-OK
@@ -183,13 +197,28 @@ export async function uploadPending() {
     }
     const data = await res.json();
     uploaded = (data.files || []);
+    if (uploaded.some(x => x && x.gallery_id)) {
+      try { localStorage.setItem('gallery-fresh-chat-upload', String(Date.now())); } catch (_) {}
+      window.dispatchEvent(new CustomEvent('gallery-refresh', { detail: { source: 'chat-upload' } }));
+    }
     pendingFiles = [];          // clear only on success
     // Stash the full meta (incl. width/height for images) on the module so
     // callers that want it can grab it via getLastUploadedMeta(). Keep the
     // returned shape as `ids` for backward-compatibility with existing call sites.
     _lastUploadedMeta = uploaded;
     return uploaded.map(x => x.id);
+  } catch (e) {
+    if (e && e.name === 'AbortError') {
+      _lastUploadCancelled = true;
+      _showToast('Upload cancelled');
+      return [];
+    }
+    _showToast('Upload failed: ' + (e?.message || 'network error'));
+    return [];
   } finally {
+    clearTimeout(timeoutId);
+    _uploading = false;
+    _uploadAbortCtrl = null;
     _uploadSpinners.forEach(sp => { try { sp.stop && sp.stop(); } catch (_) {} });
     _uploadSpinners = [];
     if (strip) strip.classList.remove('attach-uploading');
@@ -262,6 +291,7 @@ export function getPendingInfo() {
  * Clear all pending files
  */
 export function clearPending() {
+  if (_uploading) cancelUpload();
   pendingFiles.forEach(_revokePreviewUrl);
   pendingFiles = [];
   renderAttachStrip();
@@ -272,6 +302,21 @@ export function getLastUploadedMeta() {
   return _lastUploadedMeta;
 }
 
+export function isUploading() {
+  return _uploading;
+}
+
+export function wasLastUploadCancelled() {
+  return _lastUploadCancelled;
+}
+
+export function cancelUpload() {
+  _lastUploadCancelled = true;
+  if (_uploadAbortCtrl && !_uploadAbortCtrl.signal.aborted) {
+    try { _uploadAbortCtrl.abort(); } catch (_) {}
+  }
+}
+
 var escapeHtml = uiModule.esc;
 
 const fileHandlerModule = {
@@ -286,6 +331,9 @@ const fileHandlerModule = {
   getPendingRaw,
   clearPending,
   getLastUploadedMeta,
+  isUploading,
+  wasLastUploadCancelled,
+  cancelUpload,
 };
 
 export default fileHandlerModule;

static/js/gallery.js

@@ -8,13 +8,20 @@ import spinnerModule from './spinner.js';
 import { makeWindowDraggable } from './windowDrag.js';
 import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js';
 import { topPortalZ } from './toolWindowZOrder.js';
+import sessionModule from './sessions.js';
+import fileHandlerModule from './fileHandler.js';
 
 const API_BASE = window.location.origin;
 let _open = false;
 let _galleryResizeHandler = null;
 
 // Auto-refresh gallery when new image is generated
-window.addEventListener('gallery-refresh', () => {
+window.addEventListener('gallery-refresh', (e) => {
+  if (e?.detail?.source === 'chat-upload' && _sort !== 'recent') {
+    _sort = 'recent';
+    const sortSel = document.getElementById('gallery-sort');
+    if (sortSel) sortSel.value = 'recent';
+  }
   if (_open) _fetchLibrary(false);
 });
 let _items = [];
@@ -1203,7 +1210,7 @@ function _renderGrid() {
       .replace(/\.[^.]+$/, '')       // drop extension
       .replace(/[_-]+/g, ' ')
       .trim();
-    const labelText = (img.prompt || '').trim() || fallbackName || 'Photo';
+    const labelText = (img.caption || '').trim() || (img.prompt || '').trim() || fallbackName || 'Photo';
     const promptPreview = labelText.length > 60 ? labelText.substring(0, 58) + '...' : labelText;
     const favCls = img.favorite ? ' gallery-fav-active' : '';
     html += `
@@ -1351,6 +1358,10 @@ function _openDetail(img) {
         <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M17 3a2.83 2.83 0 1 1 4 4L7.5 20.5 2 22l1.5-5.5Z"/></svg>
         Edit
       </button>
+      <button class="gallery-detail-back" id="gallery-chat-photo-btn" title="${img.session_id ? 'Open source chat' : 'Start a new chat with this photo'}" aria-label="${img.session_id ? 'Open source chat' : 'Discuss photo'}" style="display:inline-flex;align-items:center;gap:4px;">
+        <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 15a4 4 0 0 1-4 4H8l-5 3V7a4 4 0 0 1 4-4h10a4 4 0 0 1 4 4z"/></svg>
+        ${img.session_id ? 'Open chat' : 'Discuss'}
+      </button>
       <button class="gallery-detail-back gallery-detail-fav-header${img.favorite ? ' active' : ''}" id="gallery-detail-fav-header" title="${img.favorite ? 'Unfavorite' : 'Favorite'}" aria-label="Favorite" aria-pressed="${img.favorite ? 'true' : 'false'}" style="display:inline-flex;align-items:center;justify-content:center;padding:4px 8px;">
         <svg width="14" height="14" viewBox="0 0 24 24" fill="${img.favorite ? 'currentColor' : 'none'}" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M20.84 4.61a5.5 5.5 0 0 0-7.78 0L12 5.67l-1.06-1.06a5.5 5.5 0 0 0-7.78 7.78l1.06 1.06L12 21.23l7.78-7.78 1.06-1.06a5.5 5.5 0 0 0 0-7.78z"/></svg>
       </button>
@@ -1412,6 +1423,7 @@ function _openDetail(img) {
             <svg class="gallery-name-enter" width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="9 10 4 15 9 20"/><path d="M20 4v7a4 4 0 0 1-4 4H4"/></svg>
           </div>
         </div>
+        ${img.caption ? `<div class="gallery-detail-section"><label>OCR Caption</label><div class="gallery-detail-prompt">${_esc(img.caption)}</div></div>` : ''}
         ${img.prompt && img.model !== 'imported' ? `<div class="gallery-detail-section"><label>Prompt</label><div class="gallery-detail-prompt">${_esc(img.prompt)}</div></div>` : ''}
         <div class="gallery-detail-section gallery-detail-section-date">
           <label>Date</label>
@@ -1456,6 +1468,45 @@ function _openDetail(img) {
     detail.style.display = 'none';
   });
 
+  document.getElementById('gallery-chat-photo-btn')?.addEventListener('click', async () => {
+    if (img.session_id) {
+      closeGallery();
+      try {
+        await sessionModule.selectSession(img.session_id);
+      } catch (e) {
+        console.error('Open source chat failed:', e);
+        uiModule.showError && uiModule.showError('Could not open source chat');
+      }
+      return;
+    }
+
+    try {
+      const dcRes = await fetch('/api/default-chat', { credentials: 'same-origin' });
+      const dc = await dcRes.json();
+      if (!dc?.endpoint_url || !dc?.model) {
+        uiModule.showError && uiModule.showError('Pick a chat model first');
+        return;
+      }
+      sessionModule.createDirectChat(dc.endpoint_url, dc.model, dc.endpoint_id);
+      closeGallery();
+      const res = await fetch(img.url, { credentials: 'same-origin' });
+      if (!res.ok) throw new Error('image fetch ' + res.status);
+      const blob = await res.blob();
+      const name = img.filename || 'gallery-photo.jpg';
+      const file = new File([blob], name, { type: blob.type || 'image/jpeg' });
+      fileHandlerModule.addFiles([file]);
+      const input = document.getElementById('message');
+      if (input) {
+        input.value = 'Let’s discuss this photo.';
+        input.dispatchEvent(new Event('input'));
+        input.focus();
+      }
+    } catch (e) {
+      console.error('Discuss photo failed:', e);
+      uiModule.showError && uiModule.showError('Could not start photo chat');
+    }
+  });
+
   // Clickable tag chips — both AI Tags and User Tags. Clicking a chip
   // closes the detail, sets the tag filter on the main grid, and
   // re-fetches so the user sees other photos with that tag.
@@ -1893,6 +1944,13 @@ export function openGallery() {
   if (_open) return;
   _open = true;
   _galleryCascaded = false;   // replay the domino-in cascade on each open
+  let _freshChatUpload = false;
+  try {
+    const ts = Number(localStorage.getItem('gallery-fresh-chat-upload') || '0');
+    _freshChatUpload = ts > 0 && Date.now() - ts < 5 * 60 * 1000;
+    if (_freshChatUpload) localStorage.removeItem('gallery-fresh-chat-upload');
+  } catch (_) {}
+  if (_freshChatUpload) _sort = 'recent';
   // State is preserved across close/reopen — filters, album, sort, items,
   // albums, people — so reopening the gallery feels instant. Use the search
   // input or "All" chip to clear the active filter.
@@ -1954,9 +2012,9 @@ export function openGallery() {
             <option value="">All sources</option>
           </select>
           <select class="gallery-sort" id="gallery-sort">
-            <option value="shuffle">Random</option>
-            <option value="recent">Recent</option>
-            <option value="oldest">Oldest</option>
+            <option value="shuffle"${_sort === 'shuffle' ? ' selected' : ''}>↭ Random order</option>
+            <option value="recent"${_sort === 'recent' ? ' selected' : ''}>↓ Newest first</option>
+            <option value="oldest"${_sort === 'oldest' ? ' selected' : ''}>↑ Oldest first</option>
           </select>
           <button class="gallery-select-btn gallery-toolbar-action" id="gallery-select-btn" title="Select for bulk actions"><span style="position:relative;top:1px;">Select</span></button>
         </div>