fix(security): harden gallery endpoint URL checks

Replace substring OpenAI endpoint detection with exact parsed-host matching.

Route gallery image endpoint construction through a constant path allowlist.

Remove client-visible exception and upstream response body leaks from gallery image flows while preserving diagnostics in server logs.

Add focused regression tests for OpenAI host matching, checked endpoint joining, harmonize SSRF hardening, and sanitized client errors.

.env.example

@@ -169,26 +169,6 @@ SEARXNG_INSTANCE=http://localhost:8080
 # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400                  # speech-to-text audio (25 MB)
 # ODYSSEUS_ICS_MAX_BYTES=10485760                        # calendar .ics import (10 MB)
 
-# ============================================================
-# Host Docker access (explicit opt-in)
-# ============================================================
-# Default Docker Compose does not mount /var/run/docker.sock. Existing
-# Ollama, vLLM, and other OpenAI-compatible endpoints remain usable without it.
-#
-# Enable this only for intentional Cookbook/local Docker-daemon management.
-# Raw socket access is high-trust and can grant broad control over the host
-# Docker daemon. Set DOCKER_GID to the host docker group's numeric GID.
-# Put these values in .env, or export them before running docker compose.
-# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
-# DOCKER_GID=963
-# docker/host-docker.yml sets this inside the container. Keep it paired
-# with the socket overlay; setting it alone is not sufficient.
-# ODYSSEUS_ENABLE_HOST_DOCKER=true
-#
-# Host Docker access can be combined with one GPU overlay:
-# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
-# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
-
 # ============================================================
 # GPU support (Docker Compose)
 # ============================================================

.github/scripts/focused_test_guidance.py

@@ -1,124 +0,0 @@
-#!/usr/bin/env python3
-"""Report focused pytest guidance for changed paths under tests/."""
-
-from __future__ import annotations
-
-import argparse
-import os
-import shlex
-import subprocess
-import sys
-from collections.abc import Iterable
-from pathlib import PurePosixPath
-
-
-def parse_paths(raw_paths: bytes) -> list[str]:
-    """Decode the NUL-delimited output of ``git diff --name-only -z``."""
-    return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path]
-
-
-def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]:
-    """Return changed ``tests/`` paths using GitHub PR three-dot semantics.
-
-    GitHub PR changed files are based on the merge base and the PR head, not a
-    direct endpoint diff between the current base branch tip and the PR head.
-    Using the direct endpoint diff can include files changed only on the base
-    branch when the PR branch is stale.
-    """
-    merge_base = subprocess.check_output(
-        ["git", "merge-base", base_sha, head_sha],
-        stderr=subprocess.DEVNULL,
-    ).strip()
-    raw_paths = subprocess.check_output(
-        [
-            "git",
-            "diff",
-            "--name-only",
-            "--diff-filter=ACMRT",
-            "-z",
-            os.fsdecode(merge_base),
-            head_sha,
-            "--",
-            "tests/",
-        ],
-    )
-    return parse_paths(raw_paths)
-
-
-def select_test_paths(paths: Iterable[str]) -> list[str]:
-    """Return unique, repository-relative paths contained by tests/."""
-    selected: set[str] = set()
-    for raw_path in paths:
-        path = PurePosixPath(raw_path)
-        if path.is_absolute() or ".." in path.parts:
-            continue
-        parts = tuple(part for part in path.parts if part != ".")
-        if len(parts) >= 2 and parts[0] == "tests":
-            selected.add(PurePosixPath(*parts).as_posix())
-    return sorted(selected)
-
-
-def is_pytest_file(path: str) -> bool:
-    """Return whether a changed path follows this repository's pytest naming."""
-    name = PurePosixPath(path).name
-    return name.endswith(".py") and (
-        name.startswith("test_") or name.endswith("_test.py")
-    )
-
-
-def pytest_command(paths: Iterable[str]) -> str:
-    """Build a copyable pytest command for changed runnable test files."""
-    command = ["python3", "-m", "pytest", "-q", *paths]
-    return shlex.join(command)
-
-
-def format_report(paths: Iterable[str]) -> str:
-    """Format focused guidance for CI logs and the workflow summary."""
-    changed_paths = select_test_paths(paths)
-    runnable_paths = [path for path in changed_paths if is_pytest_file(path)]
-    lines = ["## Focused test guidance (report-only)", ""]
-    if not changed_paths:
-        lines.append("No changed paths under `tests/`.")
-    else:
-        lines.extend(["Changed paths under `tests/`:", ""])
-        lines.extend(f"- `{path}`" for path in changed_paths)
-    lines.extend(["", "Suggested focused validation:", ""])
-    if runnable_paths:
-        lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```")
-    else:
-        lines.append("No directly runnable pytest files changed.")
-    lines.extend(
-        [
-            "",
-            "This guidance does not infer tests from source changes. "
-            "Existing blocking CI remains the source of truth.",
-        ]
-    )
-    return "\n".join(lines)
-
-
-def _parse_args(argv: list[str]) -> argparse.Namespace:
-    parser = argparse.ArgumentParser(
-        description="Report focused pytest guidance for changed tests/ paths.",
-    )
-    parser.add_argument("--base-sha", help="Pull request base commit SHA.")
-    parser.add_argument("--head-sha", help="Pull request head commit SHA.")
-    return parser.parse_args(argv)
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = _parse_args(sys.argv[1:] if argv is None else argv)
-    if bool(args.base_sha) != bool(args.head_sha):
-        raise SystemExit("--base-sha and --head-sha must be provided together")
-
-    if args.base_sha and args.head_sha:
-        paths = changed_paths_from_merge_base(args.base_sha, args.head_sha)
-    else:
-        paths = parse_paths(sys.stdin.buffer.read())
-
-    print(format_report(paths))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

.github/workflows/ci.yml

@@ -15,60 +15,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  focused-test-guidance:
-    name: Focused test guidance (report-only)
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Report changed test paths
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: |
-          report_file="$RUNNER_TEMP/focused-test-guidance.md"
-          publish_report() {
-            cat "$report_file"
-            if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
-              cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true
-            fi
-            return 0
-          }
-
-          report_unavailable() {
-            {
-              printf '%s\n\n' '## Focused test guidance unavailable (report-only)'
-              printf '%s\n\n' "$1"
-              printf '%s\n' 'Existing blocking CI remains the source of truth.'
-            } > "$report_file"
-            publish_report
-            exit 0
-          }
-
-          if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
-            report_unavailable "Pull request base/head metadata is missing."
-          fi
-
-          if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
-            report_unavailable "The pull request base commit is unavailable locally."
-          fi
-
-          if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then
-            report_unavailable "The pull request head commit is unavailable locally."
-          fi
-
-          if ! python3 .github/scripts/focused_test_guidance.py \
-            --base-sha "$BASE_SHA" \
-            --head-sha "$HEAD_SHA" > "$report_file"; then
-            report_unavailable "The focused test guidance helper could not produce a report."
-          fi
-
-          publish_report
-
   python-syntax:
     name: Python syntax (compileall)
     runs-on: ubuntu-latest

app.py

@@ -3,7 +3,6 @@ import mimetypes
 import os
 import sys
 import asyncio
-import time
 
 # On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
 # When started via `python -m uvicorn` from a terminal, uvicorn sets this
@@ -198,50 +197,7 @@ class _RequestTimeoutMiddleware(_BaseHTTPMiddleware):
             )
 
 
-class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
-    async def dispatch(self, request, call_next):
-        from src.interactive_gate import should_track_interactive_request, track_interactive_request
-
-        path = request.url.path or ""
-        if not should_track_interactive_request(path, request.method):
-            return await call_next(request)
-        async def _stop_background():
-            try:
-                await task_scheduler.stop_background_tasks_for_foreground(reason=f"foreground request {request.method} {path}")
-            except Exception:
-                logging.getLogger("app.foreground_gate").debug("foreground task stop failed", exc_info=True)
-        asyncio.create_task(_stop_background())
-        async with track_interactive_request(path, request.method):
-            return await call_next(request)
-
-
-class _SlowRequestLogMiddleware(_BaseHTTPMiddleware):
-    async def dispatch(self, request, call_next):
-        start = time.perf_counter()
-        status = 500
-        try:
-            response = await call_next(request)
-            status = getattr(response, "status_code", 0) or 0
-            return response
-        finally:
-            elapsed = time.perf_counter() - start
-            try:
-                threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75")
-            except Exception:
-                threshold = 0.75
-            if elapsed >= threshold:
-                logging.getLogger("app.slow_request").warning(
-                    "slow_request method=%s path=%s status=%s elapsed=%.3fs",
-                    request.method,
-                    request.url.path,
-                    status,
-                    elapsed,
-                )
-
-
 app.add_middleware(_RequestTimeoutMiddleware)
-app.add_middleware(_InteractiveActivityMiddleware)
-app.add_middleware(_SlowRequestLogMiddleware)
 
 # ========= AUTH =========
 from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
@@ -627,20 +583,6 @@ webhook_manager = WebhookManager(api_key_manager=api_key_manager)
 auth_router = setup_auth_routes(auth_manager)
 app.include_router(auth_router)
 
-
-@app.post("/api/activity/heartbeat")
-async def activity_heartbeat():
-    from src.interactive_gate import mark_browser_activity
-    await mark_browser_activity()
-    async def _stop_background():
-        try:
-            await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
-        except Exception:
-            logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
-    asyncio.create_task(_stop_background())
-    return {"ok": True}
-
-
 # Uploads
 from routes.upload_routes import setup_upload_routes
 upload_router, upload_cleanup_func = setup_upload_routes(upload_handler)
@@ -662,7 +604,7 @@ from routes.admin_wipe_routes import setup_admin_wipe_routes
 app.include_router(setup_admin_wipe_routes(session_manager))
 
 # Memory
-from routes.memory.memory_routes import setup_memory_routes
+from routes.memory_routes import setup_memory_routes
 memory_router = setup_memory_routes(memory_manager, session_manager, memory_vector=memory_vector)
 app.include_router(memory_router)
 from routes.skills_routes import setup_skills_routes
@@ -679,11 +621,11 @@ app.include_router(setup_chat_routes(
 ))
 
 # Research (background deep-research tasks)
-from routes.research.research_routes import setup_research_routes
+from routes.research_routes import setup_research_routes
 app.include_router(setup_research_routes(research_handler, session_manager=session_manager))
 
 # History
-from routes.history.history_routes import setup_history_routes
+from routes.history_routes import setup_history_routes
 app.include_router(setup_history_routes(session_manager))
 
 # Search
@@ -851,7 +793,7 @@ from routes.vault_routes import setup_vault_routes
 app.include_router(setup_vault_routes())
 
 # Contacts (CardDAV)
-from routes.contacts.contacts_routes import setup_contacts_routes
+from routes.contacts_routes import setup_contacts_routes
 app.include_router(setup_contacts_routes())
 
 from companion import setup_companion_routes
@@ -927,34 +869,6 @@ async def get_version():
 async def health_check() -> Dict[str, str]:
     return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()}
 
-@app.post("/api/client-perf")
-async def client_perf(request: Request):
-    """Low-volume frontend timing reports for stalls that happen before SSE logs."""
-    try:
-        data = await request.json()
-    except Exception:
-        data = {}
-    try:
-        kind = str(data.get("type") or "client").replace("\n", " ")[:80]
-        total_ms = float(data.get("total_ms") or 0)
-        stages = data.get("stages") if isinstance(data.get("stages"), list) else []
-        stage_txt = " ".join(
-            f"{str(s.get('name') or '')[:40]}={float(s.get('delta_ms') or 0):.0f}ms"
-            for s in stages[:20]
-            if isinstance(s, dict)
-        )
-        extra = str(data.get("extra") or "").replace("\n", " ")[:200]
-        logging.getLogger("app.client_perf").warning(
-            "client_perf type=%s total=%.0fms %s%s",
-            kind,
-            total_ms,
-            stage_txt,
-            f" extra={extra}" if extra else "",
-        )
-    except Exception:
-        logging.getLogger("app.client_perf").debug("client_perf log failed", exc_info=True)
-    return {"ok": True}
-
 @app.get("/api/ready")
 async def readiness_check() -> JSONResponse:
     """Readiness / integrity self-check — DB, data dir, local-first storage.
@@ -1051,59 +965,57 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
 
-    # Startup warmups are opt-in. They make later requests a little warmer, but
+    # Pre-warm the RAG tool index off the request path. Loading the local
-    # they also compete with the first seconds of real UI use on slow or busy
+    # embedding model + opening ChromaDB + indexing the built-in tools is a
-    # machines. Default to clear/idle startup and let requests warm what they use.
+    # one-time ~1-3s cost that otherwise lands on the user's FIRST message
-    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
+    # (showing up as a big `tool_selection` time). Doing it here makes the
-    if _startup_warmups_enabled:
+    # first turn as fast as subsequent ones (warm embed ≈ a few ms).
-        async def _warmup_tool_index():
+    async def _warmup_tool_index():
-            try:
+        try:
-                from src.tool_index import get_tool_index
+            from src.tool_index import get_tool_index
-                idx = await asyncio.to_thread(get_tool_index)
+            idx = await asyncio.to_thread(get_tool_index)
-                if idx:
+            if idx:
-                    await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
+                await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
-                    logger.info("[startup] Tool index pre-warmed")
+                logger.info("[startup] Tool index pre-warmed")
-            except Exception as e:
+        except Exception as e:
-                logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
+            logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
 
-        _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
+    _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
-
+    # Warmup: ping all known LLM endpoints to prime connections
-        async def _warmup_endpoints():
+    async def _warmup_endpoints():
-            try:
+        try:
-                import httpx
+            import httpx
-                urls = (
+            # model_discovery has no get_endpoints(); that call raised
-                    await asyncio.to_thread(model_discovery.warmup_ping_urls)
+            # AttributeError every run and silently disabled warmup/keepalive.
-                    if model_discovery else []
+            # Resolve the /models probe URLs via the real discovery API, off the
-                )
+            # event loop since discovery does a blocking port scan.
-                for url in urls:
+            urls = (
-                    try:
+                await asyncio.to_thread(model_discovery.warmup_ping_urls)
-                        async with httpx.AsyncClient(timeout=5.0) as client:
+                if model_discovery else []
-                            await client.get(url)
+            )
-                        logger.info(f"Warmup ping OK: {url}")
+            for url in urls:
-                    except Exception as e:
-                        logger.debug(f"Warmup ping failed for endpoint: {e}")
-            except Exception as e:
-                logger.debug(f"Warmup ping skipped: {e}")
-
-        _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
-    else:
-        logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
-
-    # Keep-alive is opt-in. The ping path performs model discovery, and when
-    # stale LAN endpoints are configured it can add periodic backend pressure
-    # that delays unrelated UI requests such as Notes/Documents.
-    _keepalive_enabled = str(os.getenv("ODYSSEUS_MODEL_KEEPALIVE", "")).lower() in {"1", "true", "yes", "on"}
-    if _keepalive_enabled:
-        async def _keepalive_loop():
-            while True:
                 try:
-                    await asyncio.sleep(60)
+                    async with httpx.AsyncClient(timeout=5.0) as client:
-                    await _warmup_endpoints()
+                        await client.get(url)
+                    logger.info(f"Warmup ping OK: {url}")
                 except Exception as e:
-                    logger.warning(f"Keepalive loop error: {e}")
+                    logger.debug(f"Warmup ping failed for endpoint: {e}")
-                    await asyncio.sleep(300)  # Back off on error
+        except Exception as e:
+            logger.debug(f"Warmup ping skipped: {e}")
 
-        _startup_tasks.append(asyncio.create_task(_keepalive_loop()))
+    _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
+
+    # Keep-alive: ping endpoints every 60 seconds to prevent cold starts
+    async def _keepalive_loop():
+        while True:
+            try:
+                await asyncio.sleep(60)
+                await _warmup_endpoints()
+            except Exception as e:
+                logger.warning(f"Keepalive loop error: {e}")
+                await asyncio.sleep(300)  # Back off on error
+
+    _startup_tasks.append(asyncio.create_task(_keepalive_loop()))
 
     async def _ensure_default_tasks():
         # Create/reconcile default automation tasks + personal assistant for every user.

core/atomic_io.py

@@ -34,8 +34,6 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) ->
 
 
 def atomic_write_text(path: str, text: str) -> None:
-    if not isinstance(text, str):
-        raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
     tmp = f"{path}.tmp.{os.getpid()}"
     with open(tmp, "w", encoding="utf-8") as f:

core/database.py

@@ -276,7 +276,6 @@ class GalleryImage(TimestampMixin, Base):
     id         = Column(String, primary_key=True, index=True)
     filename   = Column(String, nullable=False, unique=True)
     prompt     = Column(Text, nullable=False, default="")
-    caption    = Column(Text, nullable=True, default="")
     model      = Column(String, nullable=True)
     size       = Column(String, nullable=True)
     quality    = Column(String, nullable=True)
@@ -1183,29 +1182,6 @@ def _migrate_add_multiuser_owner_columns():
     _migrate_add_owner_to_table("documents", "ix_documents_owner")
 
 
-def _migrate_add_gallery_caption_column():
-    """Add OCR/vision caption storage for gallery images."""
-    import sqlite3
-    db_path = DATABASE_URL.replace("sqlite:///", "")
-    if not os.path.exists(db_path):
-        return
-    conn = None
-    try:
-        conn = sqlite3.connect(db_path)
-        columns = [row[1] for row in conn.execute("PRAGMA table_info(gallery_images)").fetchall()]
-        if columns and "caption" not in columns:
-            conn.execute("ALTER TABLE gallery_images ADD COLUMN caption TEXT DEFAULT ''")
-            conn.commit()
-            logging.getLogger(__name__).info("Migrated: added caption column to gallery_images")
-    except Exception as e:
-        logging.getLogger(__name__).warning(f"Migration gallery caption column failed: {e}")
-    finally:
-        try:
-            conn.close()
-        except Exception:
-            pass
-
-
 def _migrate_add_api_token_scopes_column():
     """Add API token scopes for existing installs.
 
@@ -1694,7 +1670,6 @@ class CalendarEvent(TimestampMixin, Base):
     # `Z`-suffix on serialization so the frontend interprets correctly.
     is_utc      = Column(Boolean, default=False, nullable=False)
     rrule       = Column(String, default="")
-    recurrence_exdates = Column(Text, default="")  # JSON list of skipped occurrence starts
     color       = Column(String, nullable=True)  # per-event color override
     status      = Column(String, default="confirmed")  # confirmed, cancelled
     importance  = Column(String, default="normal")    # low | normal | high | critical
@@ -1836,7 +1811,6 @@ def init_db():
     _migrate_add_token_columns()
     _migrate_add_mode_column()
     _migrate_add_multiuser_owner_columns()
-    _migrate_add_gallery_caption_column()
     _migrate_add_api_token_scopes_column()
     _migrate_backfill_document_owner_from_session()
     _migrate_assign_legacy_owner()
@@ -1859,7 +1833,6 @@ def init_db():
     _migrate_add_calendar_origin()
     _migrate_add_calendar_account_id()
     _migrate_add_caldav_sync_columns()
-    _migrate_add_calendar_recurrence_exdates()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2211,28 +2184,6 @@ def _migrate_add_calendar_metadata():
         except Exception:
             pass
 
-
-def _migrate_add_calendar_recurrence_exdates():
-    """Add skipped recurrence occurrences for deleting one instance of a series."""
-    import sqlite3
-    db_path = DATABASE_URL.replace("sqlite:///", "")
-    if not os.path.exists(db_path):
-        return
-    conn = None
-    try:
-        conn = sqlite3.connect(db_path)
-        columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
-        if columns and "recurrence_exdates" not in columns:
-            conn.execute("ALTER TABLE calendar_events ADD COLUMN recurrence_exdates TEXT DEFAULT ''")
-        conn.commit()
-    except Exception as e:
-        logging.getLogger(__name__).warning(f"calendar_events recurrence_exdates migration failed: {e}")
-    finally:
-        try:
-            conn.close()
-        except Exception:
-            pass
-
 def get_db():
     """
     Dependency to get a database session.

core/middleware.py

@@ -117,7 +117,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
                 f"script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net; "
                 "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
                 "font-src 'self' https://cdn.jsdelivr.net; "
-                "img-src 'self' data: blob: https:; "
+                "img-src 'self' data: blob:; "
                 "media-src 'self' blob:; "
                 "connect-src 'self'; "
                 "frame-src 'self'; "

docker-compose.gpu-amd.yml

@@ -28,6 +28,14 @@ services:
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
       - ${APP_DATA_DIR:-./data}/local:/app/.local:z
+      # Docker socket — lets Cookbook launch commands like
+      # `docker exec ollama-rocm ollama show <tag>` reach the host's
+      # Docker daemon (and sibling containers like ollama-rocm /
+      # ollama-test). The in-container user needs to be in the
+      # socket's owning group — see `group_add` below; the GID
+      # there must match the host's `docker` group (defaults to 963
+      # on Debian, 999 on Ubuntu — override via env if yours differs).
+      - /var/run/docker.sock:/var/run/docker.sock
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.
@@ -93,6 +101,7 @@ services:
       - /dev/kfd
       - /dev/dri
     group_add:
+      - "${DOCKER_GID:-963}"
       - video
       - ${RENDER_GID:-render}
 

docker-compose.gpu-nvidia.yml

@@ -27,6 +27,16 @@ services:
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
       - ${APP_DATA_DIR:-./data}/local:/app/.local:z
+      # Docker socket — lets Cookbook launch commands like
+      # `docker exec ollama-rocm ollama show <tag>` reach the host's
+      # Docker daemon (and sibling containers like ollama-rocm /
+      # ollama-test). The in-container user needs to be in the
+      # socket's owning group — see `group_add` below; the GID
+      # there must match the host's `docker` group (defaults to 963
+      # on Debian, 999 on Ubuntu — override via env if yours differs).
+      - /var/run/docker.sock:/var/run/docker.sock
+    group_add:
+      - "${DOCKER_GID:-963}"
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.

docker-compose.yml

@@ -16,6 +16,16 @@ services:
       # land under /app/.local for the odysseus user. Persist them so a
       # container recreate does not silently remove installed serve engines.
       - ${APP_DATA_DIR:-./data}/local:/app/.local:z
+      # Docker socket — lets Cookbook launch commands like
+      # `docker exec ollama-rocm ollama show <tag>` reach the host's
+      # Docker daemon (and sibling containers like ollama-rocm /
+      # ollama-test). The in-container user needs to be in the
+      # socket's owning group — see `group_add` below; the GID
+      # there must match the host's `docker` group (defaults to 963
+      # on Debian, 999 on Ubuntu — override via env if yours differs).
+      - /var/run/docker.sock:/var/run/docker.sock
+    group_add:
+      - "${DOCKER_GID:-963}"
     extra_hosts:
       # Lets the container reach local services on the Docker host, including
       # Ollama at http://host.docker.internal:11434.

docker/entrypoint.sh

@@ -29,12 +29,12 @@ fi
 ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)"
 [ -z "$ODY_USER" ] && ODY_USER=odysseus
 
-# Docker-socket group plumbing for the explicit host-Docker overlay. When
+# Docker-socket group plumbing. When /var/run/docker.sock is bind-mounted
-# opted in, the socket is owned by root:<host docker gid>. Add the app user
+# (Cookbook uses docker exec to reach sibling containers), the socket is
-# to that group and later call gosu by username so supplementary groups are
+# owned by root:<host docker gid>. Add the app user to that group and later
-# retained.
+# call gosu by username so supplementary groups are retained.
 DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}"
-if [ "${ODYSSEUS_ENABLE_HOST_DOCKER:-}" = "true" ] && [ -S "$DOCKER_SOCK" ]; then
+if [ -S "$DOCKER_SOCK" ]; then
     SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')"
     if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then
         if ! getent group "$SOCK_GID" >/dev/null 2>&1; then

docker/host-docker.yml

@@ -1,12 +0,0 @@
-# High-trust host Docker access. Enable only when local Docker-daemon
-# management from Cookbook is required and you accept that raw socket access
-# grants broad control over the host Docker daemon.
-#   COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
-#   DOCKER_GID=<numeric host Docker group id>
-services:
-  odysseus:
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    group_add: ["${DOCKER_GID:-963}"]
-    environment:
-      - ODYSSEUS_ENABLE_HOST_DOCKER=true

docs/setup.md

@@ -99,33 +99,6 @@ Odysseus SSH key and add the public key to the remote server's
 ssh-copy-id -i data/ssh/id_ed25519.pub user@server
 ```
 
-**Host Docker access (explicit opt-in).** Default Docker Compose intentionally
-does not mount `/var/run/docker.sock`. You can still connect Odysseus to
-existing Ollama, vLLM, and other OpenAI-compatible endpoints without Docker
-socket access.
-
-Cookbook/local Docker-daemon management requires the opt-in overlay below. Raw
-Docker socket access is high-trust because it can effectively grant broad
-control over the host Docker daemon. Remote server Docker workflows over SSH
-remain preferred.
-
-Place these values in `.env`, or export them in the shell before running
-`docker compose`:
-
-```bash
-COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
-DOCKER_GID=<host docker group gid>
-```
-
-Combine host Docker access with a GPU overlay when both are intentionally
-required:
-
-```bash
-COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
-# or
-COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
-```
-
 **Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
 only detect GPUs that Docker exposes to the container — if the host runtime or
 device passthrough is not configured, Cookbook sees the iGPU, another card, or
@@ -472,4 +445,4 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum
 `memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
 
 To back up or restore everything in `data/`, see the
-[Backup & Restore guide](backup-restore.md).
+[Backup & Restore guide](docs/backup-restore.md).

mcp_servers/email_server.py

@@ -58,11 +58,6 @@ def _uid_fetch_rows(data) -> list:
 _ACCOUNT_CACHE: dict = {}  # key = normalized account selector -> config dict
 _MCP_OWNER_ARG = "_odysseus_owner"
 _CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None)
-_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_EMAIL_OWNER", "ODYSSEUS_EMAIL_OWNER")
-_OWNER_SCOPE_ERROR = (
-    "Error: email MCP requires an authenticated owner or ODYSSEUS_MCP_EMAIL_OWNER "
-    "when owner-scoped email accounts are configured."
-)
 
 
 def _clean_header_value(value) -> str:
@@ -76,29 +71,13 @@ def _db_path() -> Path:
     return Path(APP_DB)
 
 
-def _configured_owner() -> str | None:
-    for key in _OWNER_ENV_KEYS:
-        owner = os.environ.get(key, "").strip()
-        if owner:
-            return owner
-    return None
-
-
 def _current_owner() -> str:
     owner = _CURRENT_OWNER.get()
-    return str(owner or _configured_owner() or "").strip()
+    return str(owner or "").strip()
-
-
-def _account_owner(row: dict) -> str:
-    return str(row.get("owner") or "").strip()
-
-
-def _has_owner_scoped_accounts(rows: list[dict]) -> bool:
-    return any(_account_owner(r) for r in rows)
 
 
 def _account_visible_to_owner(row: dict, owner: str) -> bool:
-    row_owner = _account_owner(row)
+    row_owner = str(row.get("owner") or "").strip()
     if row_owner == owner:
         return True
     if row_owner:
@@ -117,7 +96,8 @@ def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]:
     if owner:
         return [r for r in rows if _account_visible_to_owner(r, owner)]
 
-    if _has_owner_scoped_accounts(rows):
+    owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()}
+    if len(owners) > 1:
         return []
     return rows
 
@@ -126,7 +106,8 @@ def _mcp_owner_required(rows: list[dict] | None = None) -> bool:
     if _current_owner():
         return False
     rows = rows if rows is not None else _read_accounts_from_db()
-    return _has_owner_scoped_accounts(rows)
+    owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()}
+    return len(owners) > 1
 
 
 def _load_email_writing_style() -> str:
@@ -293,8 +274,6 @@ def _load_config(account: str | None = None) -> dict:
     }
 
     raw_rows = _read_accounts_from_db()
-    if _mcp_owner_required(raw_rows):
-        raise ValueError(_OWNER_SCOPE_ERROR)
     rows = _filter_accounts_for_owner(raw_rows)
     row = _resolve_account_from_rows(rows, account)
     if _current_owner() and raw_rows and not rows:
@@ -559,148 +538,6 @@ def _get_cached_summaries():
         return {}
 
 
-def _fixture_email_file() -> Path:
-    return DATA_DIR / "fixture_email_messages.json"
-
-
-def _fixture_email_enabled() -> bool:
-    return _fixture_email_file().exists()
-
-
-def _parse_fixture_date(raw_date: str) -> tuple[str, float]:
-    if not raw_date:
-        return "", 0.0
-    parsed = None
-    try:
-        parsed = datetime.fromisoformat(str(raw_date).replace("Z", "+00:00"))
-    except Exception:
-        try:
-            parsed = email.utils.parsedate_to_datetime(str(raw_date))
-        except Exception:
-            parsed = None
-    if parsed:
-        return parsed.isoformat(), parsed.timestamp()
-    return str(raw_date), 0.0
-
-
-def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict:
-    sender = str(row.get("from") or "Fixture Sender <fixture@example.invalid>")
-    sender_name, sender_addr = email.utils.parseaddr(sender)
-    date_str, date_epoch = _parse_fixture_date(str(row.get("date") or ""))
-    subject = str(row.get("subject") or "(no subject)")
-    body = str(row.get("body") or "")
-    owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default")
-    uid = str(uid_num)
-    return {
-        "uid": uid,
-        "message_id": f"<fixture-email-{uid}-{owner_key}@fixtures.odysseus.local>",
-        "subject": subject,
-        "from": sender_name or sender_addr or sender,
-        "from_address": sender_addr,
-        "date": date_str,
-        "date_epoch": date_epoch,
-        "summary": body[:240],
-        "body": body,
-        "account": "Fixture Inbox",
-        "account_email": owner or str(row.get("owner") or ""),
-        "account_id": "fixture-email",
-        "attachments": [],
-    }
-
-
-def _fixture_email_rows(owner: str | None = None) -> list[dict]:
-    path = _fixture_email_file()
-    if not path.exists():
-        return []
-    try:
-        raw = json.loads(path.read_text(encoding="utf-8"))
-    except Exception:
-        return []
-    rows = raw.get("messages") if isinstance(raw, dict) else raw
-    out = []
-    owner = str(owner or "").strip()
-    for i, row in enumerate(rows if isinstance(rows, list) else [], start=1):
-        if not isinstance(row, dict):
-            continue
-        row_owner = str(row.get("owner") or "").strip()
-        if owner and row_owner and row_owner != owner:
-            continue
-        out.append(_fixture_email_record(row, i, owner or row_owner))
-    out.sort(key=lambda item: item.get("date_epoch") or 0, reverse=True)
-    return out
-
-
-def _fixture_account_rows() -> list[dict]:
-    if not _fixture_email_enabled():
-        return []
-    owner = _current_owner()
-    owners = []
-    for row in _fixture_email_rows(owner or None):
-        email_addr = row.get("account_email") or owner or "fixture@fixtures.odysseus.local"
-        if email_addr not in owners:
-            owners.append(email_addr)
-    if not owners:
-        owners = [owner or "fixture@fixtures.odysseus.local"]
-    return [
-        {
-            "id": "fixture-email",
-            "owner": owner or owners[0],
-            "name": "Fixture Inbox",
-            "is_default": True,
-            "imap_user": owners[0],
-            "from_address": owners[0],
-        }
-    ]
-
-
-def _fixture_email_matches(item: dict, query: str) -> bool:
-    if not query:
-        return True
-    terms = [term for term in re.split(r"\W+", str(query).lower()) if term]
-    haystack = "\n".join(
-        str(item.get(key) or "")
-        for key in ("subject", "from", "from_address", "body", "summary")
-    ).lower()
-    return all(term in haystack for term in terms)
-
-
-def _fixture_list_emails(folder="INBOX", max_results=20, unresponded_only=False,
-                         unread_only=False, account=None) -> list[dict] | None:
-    if not _fixture_email_enabled():
-        return None
-    if account and str(account).strip().lower() not in {
-        "fixture-email",
-        "fixture inbox",
-        "fixture",
-        str(_current_owner()).lower(),
-    }:
-        return []
-    if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
-        return []
-    return _fixture_email_rows(_current_owner())[: int(max_results or 20)]
-
-
-def _fixture_search_emails(query, folders=None, max_results=20, account=None) -> list[dict] | None:
-    if not _fixture_email_enabled():
-        return None
-    rows = _fixture_list_emails("INBOX", max_results=1000, account=account) or []
-    out = [dict(row, _folder="INBOX") for row in rows if _fixture_email_matches(row, str(query or ""))]
-    return out[: int(max_results or 20)]
-
-
-def _fixture_read_email(uid=None, message_id=None, folder="INBOX", account=None) -> dict | None:
-    if not _fixture_email_enabled():
-        return None
-    if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
-        return {"error": f"Email UID {uid or message_id} not found"}
-    for item in _fixture_email_rows(_current_owner()):
-        if uid and str(item.get("uid")) == str(uid):
-            return item
-        if message_id and str(item.get("message_id")) == str(message_id):
-            return item
-    return {"error": f"Email not found with UID/Message-ID: {uid or message_id}"}
-
-
 # ── Tool implementations ──
 
 
@@ -711,9 +548,6 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False,
     Pass unread_only=True and/or unresponded_only=True for attention scans.
     account selects mailbox (None = default).
     """
-    fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, account)
-    if fixture is not None:
-        return fixture
     conn = None
     try:
         conn = _imap_connect(account)
@@ -795,9 +629,6 @@ def _result_sort_time(result: dict) -> datetime:
 
 def _list_emails_across_accounts(folder="INBOX", max_results=20,
                                  unresponded_only=False, unread_only=False):
-    fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, None)
-    if fixture is not None:
-        return fixture, []
     rows = _list_accounts_raw()
     combined = []
     errors = []
@@ -831,9 +662,6 @@ def _search_emails(query, folders=None, max_results=20, account=None):
     _list_emails plus an `_folder` tag."""
     if not query or not str(query).strip():
         return []
-    fixture = _fixture_search_emails(query, folders=folders, max_results=max_results, account=account)
-    if fixture is not None:
-        return fixture
     q = str(query).replace("\\", "\\\\").replace('"', '\\"')
     # Mail clients commonly use OR FROM/SUBJECT/TEXT to match either field.
     # IMAP SEARCH OR is binary, so we nest it.
@@ -956,9 +784,6 @@ def _extract_attachment_to_disk(msg, index, target_dir):
 
 def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
     """Read full email content by UID or message-ID. account = mailbox selector."""
-    fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=account)
-    if fixture is not None:
-        return fixture
     cfg = _load_config(account)
     conn = None
     try:
@@ -1012,9 +837,6 @@ def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
 
 
 def _read_email_across_accounts(uid=None, message_id=None, folder="INBOX"):
-    fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=None)
-    if fixture is not None:
-        return fixture
     rows = _list_accounts_raw()
     matches = []
     errors = []
@@ -1214,14 +1036,10 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b
     UI. This closes the auto-send hole that let earlier models invent
     signatures and ship them to real recipients without confirmation."""
     if _read_agent_email_confirm_setting():
-        # Even confirmation-first sends must resolve the selected account now.
-        # Otherwise a caller could stage a pending draft against another
-        # owner's account selector before browser approval handles it.
-        cfg = _load_config(account)
         return _stash_agent_draft(
             to=to, subject=subject, body=body,
             in_reply_to=in_reply_to, references=references,
-            cc=cc, bcc=bcc, account=cfg.get("account_id") or account,
+            cc=cc, bcc=bcc, account=account,
         )
     send_account, cfg = _resolve_send_config(account)
     msg = EmailMessage()
@@ -1957,10 +1775,9 @@ async def list_tools() -> list[Tool]:
         Tool(
             name="reply_to_email",
             description=(
-                "Reply to an existing email by UID. This sends immediately. Do NOT use "
+                "Reply to an existing email by UID. This sends immediately; for normal "
-                "for normal 'write/draft a reply saying X' requests; use "
+                "assistant-written replies, prefer draft_email_reply so the user can "
-                "draft_email_reply so the user can review and send from Odysseus. "
+                "review and send from Odysseus. Automatically threads the reply with "
-                "Only use this when the user explicitly says to send now. Automatically threads the reply with "
                 "In-Reply-To and References headers, prefixes 'Re:' on the subject, and "
                 "uses the original sender as the recipient. Set reply_all=true to also CC "
                 "the original To/Cc recipients. For follow-up 'reply ...' requests, use "
@@ -2167,12 +1984,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
     try:
         all_db_accounts = _read_accounts_from_db()
         if _mcp_owner_required(all_db_accounts):
-            return [TextContent(type="text", text=_OWNER_SCOPE_ERROR)]
+            return [TextContent(
+                type="text",
+                text="Error: email MCP requires an authenticated owner when multiple email account owners are configured.",
+            )]
 
         if name == "list_email_accounts":
             rows = _filter_accounts_for_owner(all_db_accounts)
-            if not rows:
-                rows = _fixture_account_rows()
             if not rows:
                 if all_db_accounts and owner:
                     return [TextContent(type="text", text="No email accounts configured for this owner.")]

requirements.txt

@@ -3,9 +3,8 @@ uvicorn
 python-multipart
 python-dotenv
 httpx
-httpcore>=1.0.9,<2.0
 pydantic>=2.13.4
-pydantic-settings>=2.14.2
+pydantic-settings>=2.14.1
 SQLAlchemy
 pypdf
 beautifulsoup4

routes/calendar_routes.py

@@ -1,7 +1,6 @@
 """Calendar routes — local SQLite-backed calendar CRUD."""
 
 import logging
-import json
 import re
 import uuid
 from datetime import datetime, date, timedelta
@@ -542,7 +541,6 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "description": ev.description or "",
         "location": ev.location or "",
         "rrule": ev.rrule or "",
-        "recurrence_exdates": _recurrence_exdates(ev),
         "calendar": ev.calendar.name if ev.calendar else "",
         "calendar_href": ev.calendar_id,
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
@@ -556,28 +554,6 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
 _RRULE_EXPANSION_LIMIT = 1000
 
 
-def _recurrence_exdates(ev: CalendarEvent) -> list[str]:
-    raw = getattr(ev, "recurrence_exdates", "") or ""
-    if not raw:
-        return []
-    try:
-        values = json.loads(raw)
-    except Exception:
-        return []
-    if not isinstance(values, list):
-        return []
-    return [str(v) for v in values if isinstance(v, str) and v.strip()]
-
-
-def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
-    if "::" not in uid:
-        return ""
-    suffix = uid.split("::", 1)[1]
-    if ev.all_day:
-        return suffix[:10]
-    return suffix[:16]
-
-
 def _expand_rrule(
     ev: CalendarEvent, start: datetime, end: datetime
 ) -> List[dict]:
@@ -642,7 +618,6 @@ def _expand_rrule(
     results = []
     truncated = False
     base = _event_to_dict(ev)
-    exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
         if occ_start >= end:
@@ -663,13 +638,8 @@ def _expand_rrule(
         # Build the compound uid: {base_uid}::{date} or ::{datetime}
         if ev.all_day:
             occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}"
-            exdate_key = occ_start.strftime("%Y-%m-%d")
         else:
             occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%dT%H:%M')}"
-            exdate_key = occ_start.strftime("%Y-%m-%dT%H:%M")
-
-        if exdate_key in exdates:
-            continue
 
         d = dict(base)
         d["uid"] = occ_uid
@@ -1180,7 +1150,7 @@ def setup_calendar_routes() -> APIRouter:
             db.close()
 
     @router.delete("/events/{uid}")
-    async def delete_event(request: Request, uid: str, scope: str = "series"):
+    async def delete_event(request: Request, uid: str):
         owner = _require_user(request)
         try:
             base_uid = _resolve_base_uid(uid)
@@ -1189,22 +1159,7 @@ def setup_calendar_routes() -> APIRouter:
         db = SessionLocal()
         try:
             ev = _get_or_404_event(db, base_uid, owner)
-            is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
-            if is_occurrence_delete:
-                key = _occurrence_exdate_key(uid, ev)
-                if not key:
-                    raise HTTPException(400, "Invalid recurring occurrence uid")
-                exdates = _recurrence_exdates(ev)
-                if key not in exdates:
-                    exdates.append(key)
-                ev.recurrence_exdates = json.dumps(sorted(exdates))
-                if is_caldav:
-                    ev.caldav_sync_pending = "update"
-                db.commit()
-                if is_caldav:
-                    await _push_caldav_event_after_commit(owner, base_uid, "update")
-                return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
             db.delete(ev)

routes/chat_helpers.py

@@ -14,7 +14,6 @@ from core.database import Session as DBSession, ModelEndpoint
 from src.llm_core import normalize_model_id
 from src.endpoint_resolver import normalize_base
 from src.context_compactor import maybe_compact, trim_for_context
-from src.model_context import estimate_tokens
 from src.auth_helpers import effective_user
 from src.prompt_security import untrusted_context_message
 from routes.prefs_routes import _load_for_user as load_prefs_for_user
@@ -100,11 +99,6 @@ class ChatContext:
     uprefs: dict
     preset: PresetInfo
     preprocessed: PreprocessedMessage
-    context_trimmed: bool = False
-    context_messages_before_trim: int = 0
-    context_messages_after_trim: int = 0
-    context_tokens_before_trim: int = 0
-    context_tokens_after_trim: int = 0
     # Documents auto-created server-side during preprocess (e.g. when an
     # attached fillable PDF gets rendered into a markdown editor doc).
     # The chat route emits a doc_update SSE event for each before streaming
@@ -783,12 +777,7 @@ async def build_chat_context(
     messages, context_length, was_compacted = await maybe_compact(
         sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
     )
-    _before_trim_messages = len(messages)
-    _before_trim_tokens = estimate_tokens(messages)
     messages = trim_for_context(messages, context_length)
-    _after_trim_messages = len(messages)
-    _after_trim_tokens = estimate_tokens(messages)
-    _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
 
     return ChatContext(
         preface=preface,
@@ -802,11 +791,6 @@ async def build_chat_context(
         uprefs=uprefs,
         preset=preset,
         preprocessed=preprocessed,
-        context_trimmed=_context_trimmed,
-        context_messages_before_trim=_before_trim_messages,
-        context_messages_after_trim=_after_trim_messages,
-        context_tokens_before_trim=_before_trim_tokens,
-        context_tokens_after_trim=_after_trim_tokens,
         auto_opened_docs=auto_opened_docs,
         uploaded_files=uploaded_files,
     )

routes/chat_routes.py

@@ -3,7 +3,6 @@
 import asyncio
 import json
 import os
-import re
 import time
 import logging
 from datetime import datetime
@@ -41,13 +40,8 @@ from routes.chat_helpers import (
     clean_thinking_for_save,
     _enforce_chat_privileges,
 )
-from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent
+from src.action_intents import classify_tool_intent as _classify_tool_intent
-from src.tool_policy import (
+from src.tool_policy import build_effective_tool_policy
-    WEB_TOOL_NAMES,
-    build_effective_tool_policy,
-    is_web_search_explicitly_denied,
-    web_search_enabled_for_turn,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -69,78 +63,6 @@ def _stream_set(session_id: str, **fields) -> None:
     rec.update(fields)
 
 
-def _message_plain_text(content: Any) -> str:
-    if isinstance(content, list):
-        parts: List[str] = []
-        for block in content:
-            if isinstance(block, dict):
-                text = block.get("text")
-                if isinstance(text, str):
-                    parts.append(text)
-            elif isinstance(block, str):
-                parts.append(block)
-        return " ".join(parts)
-    return str(content or "")
-
-
-def _last_user_plain_text(messages: List[Dict[str, Any]]) -> str:
-    for msg in reversed(messages or []):
-        if msg.get("role") == "user":
-            return _message_plain_text(msg.get("content"))
-    return ""
-
-
-def _ensure_current_request_is_latest_user(messages: List[Dict[str, Any]], current_message: str) -> List[Dict[str, Any]]:
-    """Defensively keep detached streams grounded on the request that created them."""
-    current = str(current_message or "").strip()
-    if not current:
-        return messages
-    latest = _last_user_plain_text(messages).strip()
-    if latest == current or current in latest or latest in current:
-        return messages
-    logger.warning(
-        "[chat_stream] latest user context mismatch; appending current request for model call. latest=%r current=%r",
-        latest[:120],
-        current[:120],
-    )
-    repaired = list(messages or [])
-    repaired.append({"role": "user", "content": current})
-    return repaired
-
-
-_WEB_FOLLOWUP_RE = re.compile(
-    r"^\s*(?:(?:can|could|would|will)\s+you\s+)?"
-    r"(?:check|try\s+again|look(?:\s+now|\s+it\s+up)?|search(?:\s+now|\s+online|\s+it)?|"
-    r"do\s+it|again)\??\s*$",
-    re.I,
-)
-_RECENT_WEB_CONTEXT_RE = re.compile(
-    r"\b(?:weather|forecast|rain|raining|hourly|news|headlines|rate|exchange|currency|"
-    r"price|current|latest|search|look\s+up|online)\b",
-    re.I,
-)
-
-
-def _recent_session_text(sess, limit: int = 8, max_chars: int = 2000) -> str:
-    history = getattr(sess, "history", None) or getattr(sess, "_history", None) or []
-    chunks: List[str] = []
-    for msg in history[-limit:]:
-        content = getattr(msg, "content", None)
-        if content is None and isinstance(msg, dict):
-            content = msg.get("content")
-        text = _message_plain_text(content).strip()
-        if text:
-            chunks.append(text)
-    return " ".join(chunks)[-max_chars:]
-
-
-def _is_contextual_web_followup(message: str, sess) -> bool:
-    """Treat short retry/check replies as web lookups when recent context was web."""
-    if not message or not _WEB_FOLLOWUP_RE.search(message):
-        return False
-    return bool(_RECENT_WEB_CONTEXT_RE.search(_recent_session_text(sess)))
-
-
 def _resolve_request_workspace(request, raw_value) -> tuple:
     """Resolve the posted workspace for this request: (workspace, rejected).
 
@@ -588,7 +510,6 @@ def setup_chat_routes(
         # below). Skill extraction should only learn from real agent sessions,
         # not chats we quietly promoted for a notes/calendar intent.
         user_requested_agent = (chat_mode == "agent")
-        _search_enabled = web_search_enabled_for_turn(allow_web_search, use_web)
         # Intent auto-escalation: if the user is clearly asking the assistant
         # to create a todo, reminder, or calendar event, promote chat → agent
         # for this turn so the LLM has access to manage_notes / manage_calendar.
@@ -606,10 +527,6 @@ def setup_chat_routes(
                 _tool_intent.category,
                 _tool_intent.reason,
             )
-        elif chat_mode == "chat" and _search_enabled:
-            chat_mode = "agent"
-            auto_escalated = True
-            logger.info("chat→agent auto-escalation: search enabled")
         active_doc_id = form_data.get("active_doc_id", "").strip()
         logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}")
 
@@ -702,20 +619,6 @@ def setup_chat_routes(
                     400,
                     "No model selected for this chat. Open the model picker and choose one before sending.",
                 )
-            if (
-                chat_mode == "chat"
-                and isinstance(message, str)
-                and (not _tool_intent or not _tool_intent.needs_tools)
-                and _is_contextual_web_followup(message, sess)
-            ):
-                _tool_intent = ToolIntent(True, "web", "contextual web lookup follow-up")
-                chat_mode = "agent"
-                auto_escalated = True
-                logger.info(
-                    "chat→agent auto-escalation: category=%s reason=%s",
-                    _tool_intent.category,
-                    _tool_intent.reason,
-                )
         except SessionNotFoundError as e:
             raise HTTPException(404, str(e))
         except (ValueError, ValidationError):
@@ -826,15 +729,6 @@ def setup_chat_routes(
                         logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}")
                 else:
                     logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}")
-            if not active_doc:
-                _email_doc_q = _doc_db.query(DBDocument).filter(
-                    DBDocument.session_id == session,
-                    DBDocument.is_active == True,
-                    DBDocument.language == "email",
-                )
-                active_doc = _owner_session_filter(_email_doc_q, ctx.user).order_by(DBDocument.updated_at.desc()).first()
-                if active_doc:
-                    logger.info(f"[doc-inject] found email draft by session fallback: title={active_doc.title!r}")
             if not active_doc:
                 _session_doc_q = _doc_db.query(DBDocument).filter(
                     DBDocument.session_id == session,
@@ -872,35 +766,20 @@ def setup_chat_routes(
 
         # Build disabled-tools set from frontend toggles + user privileges
         disabled_tools = set()
-        # Only disable bash when the caller *explicitly* set it to a falsy
+        # Only disable bash/web_search when the caller *explicitly* set them
-        # value. When unset (None), defer to per-user privilege checks below.
+        # to a falsy value.  When unset (None), defer to per-user privilege
-        # Web search is per-turn opt-in: either the chat pre-search setting
+        # checks below — this lets admins with can_use_bash=True use bash
-        # (`use_web=true`) or agent web toggle (`allow_web_search=true`) must
+        # by default without having to send allow_bash in every request.
-        # explicitly enable it.
         if allow_bash is not None and str(allow_bash).lower() != "true":
             disabled_tools.add("bash")
         _explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web")
-        if is_web_search_explicitly_denied(allow_web_search) or not _search_enabled:
+        if (
-            disabled_tools.update(WEB_TOOL_NAMES)
+            allow_web_search is not None
-        if _explicit_web_intent:
+            and str(allow_web_search).lower() != "true"
-            # A direct lookup/search request should not drift into personal
+            and not _explicit_web_intent
-            # tools or shell fallbacks. It can only use web_search/web_fetch
+        ):
-            # when the request's explicit web setting enabled them.
+            disabled_tools.add("web_search")
-            disabled_tools.update({
+            disabled_tools.add("web_fetch")
-                "bash", "python",
-                "search_chats", "manage_skills", "manage_memory",
-                "read_file", "write_file", "edit_file",
-                "create_document", "edit_document", "update_document",
-                "send_email", "reply_to_email",
-                "manage_notes", "manage_calendar", "manage_tasks",
-                "api_call", "builtin_browser",
-            })
-            if _search_enabled:
-                disabled_tools.difference_update(WEB_TOOL_NAMES)
-            else:
-                disabled_tools.update(WEB_TOOL_NAMES)
-        elif _search_enabled:
-            disabled_tools.difference_update(WEB_TOOL_NAMES)
 
         # Nobody/incognito mode: deny tools that would expose the user's
         # persistent memory, past chats, or other identity-linked data.
@@ -911,19 +790,19 @@ def setup_chat_routes(
                 "manage_skills",      # skill presets tied to user
             })
 
-        # Active email reader open → strip the tools that let the agent drift
+        # Active email reader open → strip the tools that let the agent
-        # away from the visible email or skip review. The only allowed compose
+        # "drift" to a new compose: create_document (writes a fake email-
-        # path is ui_control open_email_reply, which opens the same draft editor
+        # shaped .md file) and send_email (sends fresh to a recipient the
-        # as the Reply button with the generated body pre-filled. This prevents
+        # agent invented). With those gone, the only paths left for "write
-        # the model from falling back to direct SMTP when it botches a draft
+        # email saying X" are ui_control open_email_reply (draft) and
-        # call, and prevents fake email-shaped documents.
+        # reply_to_email (immediate send) — both of which use the open
+        # email's UID. Code-level enforcement instead of relying on a
+        # prompt rule the model can ignore.
         if active_email_ctx and active_email_ctx.get("uid"):
             disabled_tools.update({
                 "create_document",
                 "send_email",
-                "reply_to_email",
                 "mcp__email__send_email",
-                "mcp__email__reply_to_email",
             })
 
         # Enforce per-user privileges
@@ -951,7 +830,11 @@ def setup_chat_routes(
         from src.settings import get_setting
         _global_disabled = get_setting("disabled_tools", [])
         if _global_disabled and isinstance(_global_disabled, list):
-            disabled_tools.update(_global_disabled)
+            explicit_web_allowed = allow_web_search is not None and str(allow_web_search).lower() == "true"
+            if explicit_web_allowed:
+                disabled_tools.update(t for t in _global_disabled if t not in {"web_search", "web_fetch"})
+            else:
+                disabled_tools.update(_global_disabled)
 
         # Light auto-escalation: the user is in chat mode and just expressed a
         # notes/calendar/email intent. Grant the relevant managers but withhold
@@ -1165,16 +1048,13 @@ def setup_chat_routes(
                     _active_streams.pop(session, None)
                     return
 
-            messages = _ensure_current_request_is_latest_user(ctx.messages, message)
+            messages = ctx.messages
 
             # Auto-compact notification
             if ctx.was_compacted:
                 yield f"data: {json.dumps({'type': 'compacted', 'context_length': ctx.context_length})}\n\n"
-            if ctx.context_trimmed and not ctx.was_compacted:
-                yield f"data: {json.dumps({'type': 'context_trimmed', 'data': {'context_length': ctx.context_length, 'messages_before': ctx.context_messages_before_trim, 'messages_after': ctx.context_messages_after_trim, 'tokens_before': ctx.context_tokens_before_trim, 'tokens_after': ctx.context_tokens_after_trim}})}\n\n"
 
             full_response = ""
-            thinking_response = ""
             last_metrics = None
 
             # Configured fallback chain for the default chat model. Tried in
@@ -1264,9 +1144,7 @@ def setup_chat_routes(
                                     # Forward them so the client can show a thinking
                                     # indicator, but don't fold them into the saved
                                     # reply (mirrors the rewrite path below).
-                                    if data.get("thinking"):
+                                    if not data.get("thinking"):
-                                        thinking_response += data["delta"]
-                                    else:
                                         full_response += data["delta"]
                                         _stream_set(session, partial=full_response)
                                     yield chunk
@@ -1286,12 +1164,6 @@ def setup_chat_routes(
                                     _reported_model = last_metrics.get("model")
                                     last_metrics["requested_model"] = _requested_model
                                     last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
-                                    if ctx.context_trimmed:
-                                        last_metrics["context_trimmed"] = True
-                                        last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
-                                        last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
-                                        last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
-                                        last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
                                     if ctx.context_length and last_metrics.get("input_tokens"):
                                         pct = min(round((last_metrics["input_tokens"] / ctx.context_length) * 100, 1), 100.0)
                                         last_metrics["context_percent"] = pct
@@ -1334,11 +1206,8 @@ def setup_chat_routes(
                                 }
                                 yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
                             if full_response:
-                                _metrics_to_save = dict(last_metrics or {})
-                                if thinking_response.strip() and not _metrics_to_save.get("thinking"):
-                                    _metrics_to_save["thinking"] = thinking_response.strip()
                                 _saved_id = save_assistant_response(
-                                    sess, session_manager, session, full_response, _metrics_to_save,
+                                    sess, session_manager, session, full_response, last_metrics,
                                     character_name=ctx.preset.character_name,
                                     web_sources=web_sources,
                                     rag_sources=ctx.rag_sources,
@@ -1351,7 +1220,7 @@ def setup_chat_routes(
                                     yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
                                 run_post_response_tasks(
                                     sess, session_manager, session, message, full_response,
-                                    _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
+                                    last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
                                     incognito=incognito, compare_mode=compare_mode,
                                     character_name=ctx.preset.character_name,
                                     owner=_user,
@@ -1403,8 +1272,8 @@ def setup_chat_routes(
                     _max_rounds = max(1, min(_max_rounds, 200))
 
                     _forced_tools = None
-                    if _search_enabled:
+                    if allow_web_search is not None and str(allow_web_search).lower() == "true":
-                        _forced_tools = set(WEB_TOOL_NAMES)
+                        _forced_tools = {"web_search", "web_fetch"}
 
                     async for chunk in stream_agent_loop(
                         sess.endpoint_url,
@@ -1437,9 +1306,7 @@ def setup_chat_routes(
                                     # Reasoning tokens arrive flagged thinking:true.
                                     # Forward them for the live indicator, but keep
                                     # them out of the saved reply (same as chat mode).
-                                    if data.get("thinking"):
+                                    if not data.get("thinking"):
-                                        thinking_response += data["delta"]
-                                    else:
                                         full_response += data["delta"]
                                         _stream_set(session, partial=full_response)
                                     yield chunk
@@ -1477,26 +1344,15 @@ def setup_chat_routes(
                                     _reported_model = last_metrics.get("model")
                                     last_metrics["requested_model"] = last_metrics.get("requested_model") or _requested_model
                                     last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
-                                    if ctx.context_trimmed:
-                                        last_metrics["context_trimmed"] = True
-                                        last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
-                                        last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
-                                        last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
-                                        last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
                                     yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
                             except json.JSONDecodeError:
                                 yield chunk
                         elif chunk.startswith("event: "):
                             yield chunk
                         elif chunk == "data: [DONE]\n\n":
-                            _has_tool_events = bool((last_metrics or {}).get("tool_events"))
+                            if full_response:
-                            if full_response or _has_tool_events:
-                                _response_to_save = full_response or "Done."
-                                _metrics_to_save = dict(last_metrics or {})
-                                if thinking_response.strip() and not _metrics_to_save.get("thinking"):
-                                    _metrics_to_save["thinking"] = thinking_response.strip()
                                 _saved_id = save_assistant_response(
-                                    sess, session_manager, session, _response_to_save, _metrics_to_save,
+                                    sess, session_manager, session, full_response, last_metrics,
                                     character_name=ctx.preset.character_name,
                                     web_sources=web_sources,
                                     rag_sources=ctx.rag_sources,
@@ -1506,8 +1362,8 @@ def setup_chat_routes(
                                 if _saved_id:
                                     yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
                                 run_post_response_tasks(
-                                    sess, session_manager, session, message, _response_to_save,
+                                    sess, session_manager, session, message, full_response,
-                                    _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
+                                    last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
                                     incognito=incognito, compare_mode=compare_mode,
                                     character_name=ctx.preset.character_name,
                                                             agent_rounds=_agent_rounds,

routes/contacts/__init__.py

@@ -1,5 +0,0 @@
-"""Contacts route domain package (slice 2e, #4082/#4071).
-
-Contains contacts_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/contacts_routes.py re-exports from here.
-"""

routes/contacts/contacts_routes.py

@@ -1,916 +0,0 @@
-"""
-contacts_routes.py
-
-CardDAV contacts integration. Reads from local Radicale, supports
-search and adding new contacts.
-"""
-
-import re
-import logging
-import uuid
-import json
-import csv
-import io
-import os
-import inspect
-import httpx
-from pathlib import Path
-from datetime import datetime
-from urllib.parse import urljoin, urlparse, urlunparse
-
-from core.log_safety import redact_url
-from fastapi import APIRouter, Query, Depends, Response, HTTPException
-from typing import List, Dict, Optional
-
-from core.middleware import require_admin
-from src.url_safety import check_outbound_url
-
-logger = logging.getLogger(__name__)
-
-from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
-DATA_DIR = Path(_DATA_DIR)
-SETTINGS_FILE = Path(_SETTINGS_FILE)
-LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
-
-
-def _load_settings():
-    if SETTINGS_FILE.exists():
-        return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
-    return {}
-
-
-def _save_settings(settings):
-    from core.atomic_io import atomic_write_json
-    atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
-
-
-def _get_carddav_config():
-    import os
-    settings = _load_settings()
-    password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
-    if password and "carddav_password" in settings:
-        from src.secret_storage import decrypt
-        password = decrypt(password)
-    return {
-        "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
-        "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
-        "password": password,
-    }
-
-
-def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
-    cfg = cfg or _get_carddav_config()
-    return bool((cfg.get("url") or "").strip())
-
-
-def _validate_carddav_url(url: str) -> str:
-    cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
-    ok, reason = check_outbound_url(
-        cleaned,
-        block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
-    )
-    if not ok:
-        raise ValueError(f"Rejected CardDAV URL: {reason}")
-    return cleaned
-
-
-def _carddav_base_url(cfg: Dict) -> str:
-    return _validate_carddav_url(cfg.get("url") or "")
-
-
-def _normalize_contact(contact: Dict) -> Dict:
-    emails = []
-    for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
-        e = str(e or "").strip()
-        if e and e not in emails:
-            emails.append(e)
-    phones = []
-    for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
-        p = str(p or "").strip()
-        if p and p not in phones:
-            phones.append(p)
-    name = str(contact.get("name") or "").strip()
-    if not name and emails:
-        name = emails[0].split("@")[0]
-    address = str(contact.get("address") or "").strip()
-    return {
-        "uid": str(contact.get("uid") or uuid.uuid4()),
-        "name": name,
-        "emails": emails,
-        "phones": phones,
-        "address": address,
-    }
-
-
-def _load_local_contacts() -> List[Dict]:
-    try:
-        if not LOCAL_CONTACTS_FILE.exists():
-            return []
-        data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
-        rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
-    except Exception as e:
-        logger.error(f"Failed to load local contacts: {e}")
-        return []
-
-
-def _save_local_contacts(contacts: List[Dict]) -> None:
-    from core.atomic_io import atomic_write_json
-    DATA_DIR.mkdir(parents=True, exist_ok=True)
-    atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
-    _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
-    _contact_cache["fetched_at"] = datetime.utcnow()
-
-
-# ── vCard parsing ──
-
-def _vunesc(value: str) -> str:
-    """Reverse _vesc() — turn escaped vCard text back into the raw value.
-    Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
-    if not value:
-        return value
-    out = []
-    i = 0
-    while i < len(value):
-        ch = value[i]
-        if ch == "\\" and i + 1 < len(value):
-            nxt = value[i + 1]
-            if nxt in ("n", "N"):
-                out.append("\n")
-            elif nxt in (",", ";", "\\"):
-                out.append(nxt)
-            else:
-                out.append(nxt)
-            i += 2
-        else:
-            out.append(ch)
-            i += 1
-    return "".join(out)
-
-
-def _parse_vcards(text: str) -> List[Dict]:
-    """Parse a stream of vCards into dicts with name, email, phone."""
-    # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
-    # space or tab is a continuation of the previous logical line. Real
-    # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
-    # PHOTO lines, and splitting on raw newlines without unfolding dropped the
-    # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
-    # email/name.
-    text = re.sub(r"\r\n[ \t]", "", text or "")
-    text = re.sub(r"\n[ \t]", "", text)
-    contacts = []
-    for block in re.split(r"BEGIN:VCARD", text):
-        if not block.strip():
-            continue
-        contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
-        for line in block.split("\n"):
-            line = line.strip()
-            # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
-            # that Apple Contacts / iCloud / many CardDAV servers emit by
-            # default — without this the property-name checks below miss those
-            # lines and silently drop the email / phone. The group token only
-            # precedes the property name, so it is safe to strip for matching
-            # and value extraction, and a no-op for non-grouped lines.
-            name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
-            if name_part.startswith("FN:") or name_part.startswith("FN;"):
-                contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
-            elif name_part.startswith("EMAIL"):
-                # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
-                if ":" in name_part:
-                    email_addr = _vunesc(name_part.split(":", 1)[1])
-                    if email_addr and email_addr not in contact["emails"]:
-                        contact["emails"].append(email_addr)
-            elif name_part.startswith("TEL"):
-                if ":" in name_part:
-                    phone = _vunesc(name_part.split(":", 1)[1])
-                    if phone and phone not in contact["phones"]:
-                        contact["phones"].append(phone)
-            elif name_part.startswith("ADR"):
-                # vCard ADR is 7 semicolon-separated components:
-                # post-office-box;extended-address;street;locality;region;postal-code;country.
-                # Recover a human-readable string by joining non-empty
-                # components with ", ".
-                if ":" in name_part:
-                    raw = name_part.split(":", 1)[1]
-                    parts = [_vunesc(p).strip() for p in raw.split(";")]
-                    contact["address"] = ", ".join(p for p in parts if p)
-            elif name_part.startswith("UID:"):
-                contact["uid"] = _vunesc(name_part[4:])
-        if contact["name"] or contact["emails"]:
-            contacts.append(contact)
-    return contacts
-
-
-def _vesc(value: str) -> str:
-    """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
-    semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
-    or any value containing a newline produces a malformed vCard (broken
-    N/FN fields) or could inject arbitrary properties."""
-    return (
-        (value or "")
-        .replace("\\", "\\\\")
-        .replace("\n", "\\n")
-        .replace("\r", "")
-        .replace(",", "\\,")
-        .replace(";", "\\;")
-    )
-
-
-def _build_vcard(name: str, email: str, uid: Optional[str] = None,
-                 emails: Optional[List[str]] = None,
-                 phones: Optional[List[str]] = None,
-                 address: Optional[str] = None) -> str:
-    """Build a vCard. Accepts either a single `email` (legacy callers) or
-    full `emails`/`phones` lists (edit path). The first email is marked
-    PREF=1. All values are RFC-6350-escaped."""
-    if not uid:
-        uid = str(uuid.uuid4())
-    # Normalize email lists — `email` arg is a convenience for single-email
-    # creation; `emails` (if given) is authoritative.
-    email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
-    phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
-    # Try to split name into first/last
-    parts = name.strip().split()
-    if len(parts) >= 2:
-        first = parts[0]
-        last = " ".join(parts[1:])
-    else:
-        first = name
-        last = ""
-    # N field is structured (5 components separated by ';') — escape each
-    # component individually so a comma in the name doesn't split it.
-    n_field = f"{_vesc(last)};{_vesc(first)};;;"
-    lines = [
-        "BEGIN:VCARD",
-        "VERSION:4.0",
-        f"UID:{_vesc(uid)}",
-        f"FN:{_vesc(name)}",
-        f"N:{n_field}",
-    ]
-    for i, em in enumerate(email_list):
-        # First email is the preferred one.
-        lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
-    for ph in phone_list:
-        lines.append(f"TEL:{_vesc(ph)}")
-    # Address: stuff the whole human-readable string into the street
-    # component of ADR. vCard ADR has 7 semicolon-separated components:
-    # post-office-box;extended-address;street;locality;region;postal-code;country.
-    addr = (address or "").strip()
-    if addr:
-        lines.append(f"ADR:;;{_vesc(addr)};;;;")
-    lines.append("END:VCARD")
-    return "\r\n".join(lines) + "\r\n"
-
-
-# ── In-memory cache ──
-
-_contact_cache = {"contacts": [], "fetched_at": None}
-
-
-def _abs_url(href: str) -> str:
-    """Combine a multistatus <href> (an absolute path like
-    /user/contacts/x.vcf) with the configured CardDAV server origin so we
-    get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
-    for the configured origin; a cross-origin href is treated as a path on the
-    configured server so a malicious CardDAV response cannot redirect later
-    writes/deletes to cloud metadata or another host."""
-    cfg = _get_carddav_config()
-    base = _carddav_base_url(cfg)
-    base_p = urlparse(base)
-    joined = urljoin(base.rstrip("/") + "/", href or "")
-    joined_p = urlparse(joined)
-    if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
-        joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
-    return _validate_carddav_url(joined)
-
-
-# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
-# alongside the resource href. Lets us map each contact's UID to the real
-# server resource path (which is NOT always <uid>.vcf for contacts created
-# by other clients).
-_ADDRESSBOOK_QUERY = (
-    '<?xml version="1.0" encoding="utf-8"?>'
-    '<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
-    '<D:prop><D:getetag/><C:address-data/></D:prop>'
-    '<C:filter/>'
-    '</C:addressbook-query>'
-)
-
-
-def _fetch_via_report(cfg, auth):
-    """Try a CardDAV REPORT addressbook-query — returns contacts WITH an
-    `href` field, or None if the server doesn't support it / errors."""
-    from defusedxml import ElementTree as ET
-    try:
-        r = httpx.request(
-            "REPORT", cfg["url"],
-            content=_ADDRESSBOOK_QUERY.encode("utf-8"),
-            headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
-        )
-        if r.status_code not in (207, 200):
-            return None
-        root = ET.fromstring(r.text)
-        ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
-        out = []
-        for resp in root.findall("D:response", ns):
-            href_el = resp.find("D:href", ns)
-            data_el = resp.find(".//C:address-data", ns)
-            if href_el is None or data_el is None or not (data_el.text or "").strip():
-                continue
-            parsed = _parse_vcards(data_el.text)
-            if not parsed:
-                continue
-            c = parsed[0]
-            c["href"] = href_el.text.strip()
-            out.append(c)
-        # If the REPORT parsed to ZERO contacts, don't trust it — some
-        # CardDAV servers treat an empty <filter/> as "match nothing" and
-        # return a valid-but-empty 207. Return None so the caller falls
-        # back to the plain GET (which lists everything). A genuinely empty
-        # address book just costs one extra GET that also returns nothing.
-        if not out:
-            return None
-        return out
-    except Exception as e:
-        logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
-        return None
-
-
-def _fetch_contacts(force=False):
-    """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
-        age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
-            return _contact_cache["contacts"]
-
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
-        return contacts
-
-    try:
-        cfg["url"] = _carddav_base_url(cfg)
-        auth = None
-        if cfg["username"]:
-            auth = (cfg["username"], cfg["password"])
-        # Preferred path: REPORT gives us hrefs for reliable edit/delete.
-        contacts = _fetch_via_report(cfg, auth)
-        if contacts is None:
-            # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
-            if r.status_code != 200:
-                logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
-            contacts = _parse_vcards(r.text)
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
-        return contacts
-    except Exception as e:
-        logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
-
-
-def _resolve_resource_url(uid: str) -> str:
-    """Map a contact UID to its real CardDAV resource URL. Uses the href
-    captured during fetch when available (handles contacts whose filename
-    != UID); falls back to the <uid>.vcf guess for app-created contacts or
-    when no href is known."""
-    def _lookup():
-        for c in _contact_cache.get("contacts", []):
-            if c.get("uid") == uid and c.get("href"):
-                return _abs_url(c["href"])
-        return None
-    found = _lookup()
-    if found:
-        return found
-    # Not in cache (or no href) — refresh once and retry before guessing.
-    try:
-        _fetch_contacts(force=True)
-    except Exception:
-        pass
-    return _lookup() or _vcard_url(uid)
-
-
-def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
-    """Add a new contact via CardDAV or local contacts."""
-    email = (email or "").strip()
-    phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        email_l = email.lower()
-        for c in contacts:
-            if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
-                return True
-            if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
-                return True
-        contacts.append(_normalize_contact({
-            "name": name,
-            "emails": [email] if email else [],
-            "phones": phone_list,
-            "address": address,
-        }))
-        _save_local_contacts(contacts)
-        return True
-
-    contact_uid = str(uuid.uuid4())
-    vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
-    try:
-        url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
-        auth = None
-        if cfg["username"]:
-            auth = (cfg["username"], cfg["password"])
-        r = httpx.put(
-            url,
-            data=vcard.encode("utf-8"),
-            headers={"Content-Type": "text/vcard; charset=utf-8"},
-            auth=auth,
-            timeout=10,
-        )
-        if r.status_code in (200, 201, 204):
-            # Invalidate cache
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to create contact: {e}")
-        return False
-
-
-def _vcard_url(uid: str) -> str:
-    """The CardDAV resource URL for a given contact UID. The uid is URL-
-    encoded so a value containing '/', '..' or other path chars can't
-    escape the collection and target an arbitrary CardDAV resource."""
-    from urllib.parse import quote
-    cfg = _get_carddav_config()
-    return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
-
-
-def _import_vcards(text: str) -> Dict:
-    """Import a (possibly multi-card) .vcf blob. Each card is PUT to the
-    CardDAV server PRESERVING its full original content (ADR/ORG/photo/
-    etc.) — we don't rebuild it, just ensure it has VERSION + UID and
-    normalize line endings. Returns {imported, failed, total}."""
-    from urllib.parse import quote
-    cfg = _get_carddav_config()
-    if not cfg.get("url"):
-        parsed = _parse_vcards(text)
-        contacts = _load_local_contacts()
-        existing = {
-            e.lower()
-            for c in contacts
-            for e in (c.get("emails") or [])
-            if e
-        }
-        imported = 0
-        for c in parsed:
-            emails = [e for e in (c.get("emails") or []) if e]
-            if emails and any(e.lower() in existing for e in emails):
-                continue
-            contacts.append(_normalize_contact(c))
-            for e in emails:
-                existing.add(e.lower())
-            imported += 1
-        if imported:
-            _save_local_contacts(contacts)
-        return {"imported": imported, "failed": 0, "total": len(parsed)}
-    try:
-        base_url = _carddav_base_url(cfg)
-    except ValueError as e:
-        logger.warning("CardDAV import URL rejected: %s", e)
-        return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
-    auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-    # Split into individual cards. re.split drops the BEGIN line, so we
-    # re-add it. Normalize CRLF.
-    raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
-    blocks = []
-    for chunk in raw.split("BEGIN:VCARD"):
-        chunk = chunk.strip()
-        if not chunk:
-            continue
-        # Trim anything after END:VCARD (defensive).
-        end = chunk.upper().find("END:VCARD")
-        body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
-        blocks.append("BEGIN:VCARD\n" + body)
-    imported = 0
-    failed = 0
-    for block in blocks:
-        # Extract or assign a UID.
-        m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
-        uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
-        if not m:
-            # Inject a UID right after the VERSION line (or after BEGIN).
-            if re.search(r"^VERSION:", block, re.MULTILINE):
-                block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
-            else:
-                block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
-        elif not re.search(r"^VERSION:", block, re.MULTILINE):
-            block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
-        vcard = block.replace("\n", "\r\n") + "\r\n"
-        url = base_url + "/" + quote(uid, safe="") + ".vcf"
-        try:
-            r = httpx.put(
-                url, data=vcard.encode("utf-8"),
-                headers={"Content-Type": "text/vcard; charset=utf-8"},
-                auth=auth, timeout=15,
-            )
-            if r.status_code in (200, 201, 204):
-                imported += 1
-            else:
-                failed += 1
-                logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
-        except Exception as e:
-            failed += 1
-            logger.error(f"Import PUT {uid} failed: {e}")
-    if imported:
-        _contact_cache["fetched_at"] = None
-    return {"imported": imported, "failed": failed, "total": len(blocks)}
-
-
-def _import_csv_contacts(text: str) -> Dict:
-    """Import contacts from CSV. Supports common headers:
-    name/full_name/display_name, email/email_address/e-mail, phone/tel.
-    Falls back to first columns as name,email,phone when no headers exist."""
-    raw = (text or "").strip()
-    if not raw:
-        return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
-
-    try:
-        sample = raw[:2048]
-        dialect = csv.Sniffer().sniff(sample)
-    except Exception:
-        dialect = csv.excel
-
-    stream = io.StringIO(raw)
-    try:
-        has_header = csv.Sniffer().has_header(raw[:2048])
-    except Exception:
-        has_header = True
-
-    rows = []
-    if has_header:
-        reader = csv.DictReader(stream, dialect=dialect)
-        for row in reader:
-            lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
-            name = (
-                lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
-                or lowered.get("display name") or lowered.get("display_name")
-                or lowered.get("fn") or ""
-            )
-            email = (
-                lowered.get("email") or lowered.get("email address")
-                or lowered.get("email_address") or lowered.get("e-mail")
-                or lowered.get("mail") or ""
-            )
-            phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
-            rows.append((name, email, phone))
-    else:
-        stream.seek(0)
-        reader = csv.reader(stream, dialect=dialect)
-        for row in reader:
-            cols = [(c or "").strip() for c in row]
-            if not any(cols):
-                continue
-            rows.append((
-                cols[0] if len(cols) > 0 else "",
-                cols[1] if len(cols) > 1 else "",
-                cols[2] if len(cols) > 2 else "",
-            ))
-
-    imported = 0
-    failed = 0
-    total = 0
-    existing_emails = {
-        e.lower()
-        for c in _fetch_contacts()
-        for e in (c.get("emails") or [])
-        if e
-    }
-    for name, email, phone in rows:
-        email = (email or "").strip()
-        name = (name or "").strip() or (email.split("@")[0] if email else "")
-        if not email:
-            continue
-        total += 1
-        if email.lower() in existing_emails:
-            continue
-        ok = _create_contact(name, email)
-        if ok:
-            imported += 1
-            existing_emails.add(email.lower())
-            # If the CSV had a phone number, rewrite the just-created row
-            # through the richer update path so phone lands in CardDAV too.
-            if phone:
-                try:
-                    contacts = _fetch_contacts(force=True)
-                    created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
-                    if created and created.get("uid"):
-                        _update_contact(created["uid"], name, [email], [phone])
-                except Exception:
-                    pass
-        else:
-            failed += 1
-
-    if imported:
-        _contact_cache["fetched_at"] = None
-    return {"imported": imported, "failed": failed, "total": total}
-
-
-def _contacts_to_vcf(contacts: List[Dict]) -> str:
-    return "".join(
-        _build_vcard(
-            c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
-            "",
-            uid=c.get("uid") or str(uuid.uuid4()),
-            emails=c.get("emails") or [],
-            phones=c.get("phones") or [],
-        )
-        for c in contacts
-    )
-
-
-def _contacts_to_csv(contacts: List[Dict]) -> str:
-    out = io.StringIO()
-    writer = csv.writer(out)
-    writer.writerow(["name", "email", "phone"])
-    for c in contacts:
-        emails = c.get("emails") or [""]
-        phones = c.get("phones") or [""]
-        max_len = max(len(emails), len(phones), 1)
-        for i in range(max_len):
-            writer.writerow([
-                c.get("name") or "",
-                emails[i] if i < len(emails) else "",
-                phones[i] if i < len(phones) else "",
-            ])
-    return out.getvalue()
-
-
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
-    """Rewrite an existing contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        found = False
-        out = []
-        for c in contacts:
-            if c.get("uid") == uid:
-                # Preserve existing address when caller passes "" (only
-                # updating name/emails/phones, not touching address).
-                addr = address if address else c.get("address", "")
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
-                found = True
-            else:
-                out.append(c)
-        if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
-        _save_local_contacts(out)
-        return True
-
-    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
-    # Use the real resource href (handles externally-created contacts whose
-    # filename != UID); falls back to the <uid>.vcf guess.
-    try:
-        url = _resolve_resource_url(uid)
-        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-        r = httpx.put(
-            url,
-            data=vcard.encode("utf-8"),
-            headers={"Content-Type": "text/vcard; charset=utf-8"},
-            auth=auth,
-            timeout=10,
-        )
-        if r.status_code in (200, 201, 204):
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to update contact: {e}")
-        return False
-
-
-def _delete_contact(uid: str) -> bool:
-    """Delete a contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
-        _save_local_contacts(remaining)
-        return True
-
-    try:
-        url = _resolve_resource_url(uid)
-        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-        r = httpx.delete(url, auth=auth, timeout=10)
-        if r.status_code in (200, 204, 404):
-            # Invalidate cache so the next fetch sees the server truth.
-            _contact_cache["fetched_at"] = None
-            # Verify: force a fresh fetch and check the UID is actually gone.
-            # A 404 on the guessed URL ({uid}.vcf) can mean the contact
-            # lives at a different resource URL — the DELETE missed it but
-            # we'd silently report success. This check catches that.
-            fresh = _fetch_contacts(force=True)
-            still_there = any(c.get("uid") == uid for c in fresh)
-            if still_there:
-                logger.warning(
-                    f"CardDAV DELETE reported success for {uid} "
-                    f"but UID still present after re-fetch — "
-                    f"resource URL may differ from {redact_url(url)}"
-                )
-                return False
-            if r.status_code == 404:
-                logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
-            return True
-        logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to delete contact: {e}")
-        return False
-
-
-# ── Routes ──
-
-def setup_contacts_routes():
-    router = APIRouter(prefix="/api/contacts", tags=["contacts"])
-
-    @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
-        """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
-
-    @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
-        """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
-        if not q:
-            return {"results": []}
-        q_lower = q.lower()
-        results = []
-        for c in contacts:
-            if q_lower in c["name"].lower():
-                results.append(c)
-                continue
-            for em in c["emails"]:
-                if q_lower in em.lower():
-                    results.append(c)
-                    break
-        return {"results": results[:10]}
-
-    @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
-        """Add a new contact."""
-        name = (data.get("name") or "").strip()
-        email = (data.get("email") or "").strip()
-        phone = (data.get("phone") or "").strip()
-        phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
-        if phone and phone not in phones:
-            phones.insert(0, phone)
-        address = (data.get("address") or "").strip()
-        if not name and email:
-            name = email.split("@")[0]
-        if not name and not email and not phones and not address:
-            return {"success": False, "error": "Name, email, phone, or address required"}
-        if not name:
-            name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
-        contacts = _fetch_contacts()
-        for c in contacts:
-            if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
-                return {"success": True, "message": "Already exists", "contact": c}
-            if phones and any(p in (c.get("phones") or []) for p in phones):
-                return {"success": True, "message": "Already exists", "contact": c}
-        create_params = inspect.signature(_create_contact).parameters
-        if "phones" in create_params:
-            ok = _create_contact(name, email, address, phones=phones)
-        elif len(create_params) >= 3:
-            ok = _create_contact(name, email, address)
-        else:
-            ok = _create_contact(name, email)
-        # If a phone was provided, do an immediate update to thread it
-        # through (the simple _create_contact signature only takes name +
-        # email + address; phones happen via update).
-        if ok and phones and "phones" not in create_params:
-            try:
-                fresh = _fetch_contacts(force=True)
-                created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
-                if created:
-                    _update_contact(
-                        created["uid"], name,
-                        created.get("emails", []),
-                        phones,
-                        address,
-                    )
-            except Exception:
-                pass
-        return {"success": ok}
-
-    @router.post("/import")
-    async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
-        """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
-        # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
-        # in the JSON body) would otherwise reach .strip() and 500 with an
-        # AttributeError instead of degrading to a clean "no data" response.
-        text = str(data.get("vcf") or data.get("text") or "")
-        csv_text = str(data.get("csv") or "")
-        if text.strip():
-            if "BEGIN:VCARD" not in text.upper():
-                return {"success": False, "error": "No vCard data found"}
-            result = _import_vcards(text)
-        elif csv_text.strip():
-            result = _import_csv_contacts(csv_text)
-        else:
-            return {"success": False, "error": "No contact data found"}
-        result["success"] = result.get("imported", 0) > 0
-        return result
-
-    @router.get("/export")
-    async def export_contacts(
-        format: str = Query("vcf", pattern="^(vcf|csv)$"),
-        _admin: str = Depends(require_admin),
-    ):
-        """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
-        if format == "csv":
-            content = _contacts_to_csv(contacts)
-            media_type = "text/csv; charset=utf-8"
-            filename = "odysseus-contacts.csv"
-        else:
-            content = _contacts_to_vcf(contacts)
-            media_type = "text/vcard; charset=utf-8"
-            filename = "odysseus-contacts.vcf"
-        return Response(
-            content=content,
-            media_type=media_type,
-            headers={"Content-Disposition": f'attachment; filename="{filename}"'},
-        )
-
-    @router.get("/config")
-    async def get_config(_admin: str = Depends(require_admin)):
-        cfg = _get_carddav_config()
-        # Mask password
-        if cfg["password"]:
-            cfg["password"] = "***"
-        return cfg
-
-    @router.put("/config")
-    async def update_config(data: dict, _admin: str = Depends(require_admin)):
-        settings = _load_settings()
-        for key in ("carddav_url", "carddav_username", "carddav_password"):
-            if key in data:
-                if key == "carddav_url" and str(data[key] or "").strip():
-                    try:
-                        settings[key] = _validate_carddav_url(data[key])
-                    except ValueError as e:
-                        raise HTTPException(400, str(e))
-                else:
-                    value = data[key]
-                    if key == "carddav_password" and value:
-                        from src.secret_storage import encrypt
-                        value = encrypt(value)
-                    settings[key] = value
-        _save_settings(settings)
-        # Force re-fetch
-        _contact_cache["fetched_at"] = None
-        return {"success": True}
-
-    @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
-        """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
-        return {"success": True}
-
-    # NOTE: the /{uid} routes are declared LAST so the literal paths above
-    # (/list, /search, /add, /config) win — otherwise PUT /config would
-    # match PUT /{uid} with uid="config".
-    @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
-        """Edit an existing contact — name / emails / phones / address."""
-        name = (data.get("name") or "").strip()
-        emails = data.get("emails")
-        phones = data.get("phones")
-        if emails is None and data.get("email"):
-            emails = [data["email"]]
-        emails = [e.strip() for e in (emails or []) if e and e.strip()]
-        phones = [p.strip() for p in (phones or []) if p and p.strip()]
-        address = (data.get("address") or "").strip()
-        if not name and not emails and not address:
-            return {"success": False, "error": "Name, email, or address required"}
-        if not name and emails:
-            name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones, address)
-        return {"success": ok}
-
-    @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
-        """Delete a contact by UID."""
-        if not uid:
-            return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
-        return {"success": ok}
-
-    return router

routes/contacts_routes.py

@@ -1,13 +1,900 @@
-"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py.
+"""
+contacts_routes.py
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+CardDAV contacts integration. Reads from local Radicale, supports
-that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``,
+search and adding new contacts.
-``importlib.import_module("routes.contacts_routes")``, and string-targeted
-monkeypatches all operate on the same object the application actually uses.
 """
 
-import sys as _sys
+import re
+import logging
+import uuid
+import json
+import csv
+import io
+import os
+import inspect
+import httpx
+from pathlib import Path
+from datetime import datetime
+from urllib.parse import urljoin, urlparse, urlunparse
 
-from routes.contacts import contacts_routes as _canonical  # noqa: F401
+from core.log_safety import redact_url
+from fastapi import APIRouter, Query, Depends, Response, HTTPException
+from typing import List, Dict, Optional
 
-_sys.modules[__name__] = _canonical
+from core.middleware import require_admin
+from src.url_safety import check_outbound_url
+
+logger = logging.getLogger(__name__)
+
+from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
+DATA_DIR = Path(_DATA_DIR)
+SETTINGS_FILE = Path(_SETTINGS_FILE)
+LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
+
+
+def _load_settings():
+    if SETTINGS_FILE.exists():
+        return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
+    return {}
+
+
+def _save_settings(settings):
+    from core.atomic_io import atomic_write_json
+    atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
+
+
+def _get_carddav_config():
+    import os
+    settings = _load_settings()
+    password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
+    if password and "carddav_password" in settings:
+        from src.secret_storage import decrypt
+        password = decrypt(password)
+    return {
+        "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
+        "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
+        "password": password,
+    }
+
+
+def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
+    cfg = cfg or _get_carddav_config()
+    return bool((cfg.get("url") or "").strip())
+
+
+def _validate_carddav_url(url: str) -> str:
+    cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
+    ok, reason = check_outbound_url(
+        cleaned,
+        block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
+    )
+    if not ok:
+        raise ValueError(f"Rejected CardDAV URL: {reason}")
+    return cleaned
+
+
+def _carddav_base_url(cfg: Dict) -> str:
+    return _validate_carddav_url(cfg.get("url") or "")
+
+
+def _normalize_contact(contact: Dict) -> Dict:
+    emails = []
+    for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
+        e = str(e or "").strip()
+        if e and e not in emails:
+            emails.append(e)
+    phones = []
+    for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
+        p = str(p or "").strip()
+        if p and p not in phones:
+            phones.append(p)
+    name = str(contact.get("name") or "").strip()
+    if not name and emails:
+        name = emails[0].split("@")[0]
+    address = str(contact.get("address") or "").strip()
+    return {
+        "uid": str(contact.get("uid") or uuid.uuid4()),
+        "name": name,
+        "emails": emails,
+        "phones": phones,
+        "address": address,
+    }
+
+
+def _load_local_contacts() -> List[Dict]:
+    try:
+        if not LOCAL_CONTACTS_FILE.exists():
+            return []
+        data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
+        rows = data.get("contacts", data) if isinstance(data, dict) else data
+        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+    except Exception as e:
+        logger.error(f"Failed to load local contacts: {e}")
+        return []
+
+
+def _save_local_contacts(contacts: List[Dict]) -> None:
+    from core.atomic_io import atomic_write_json
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
+    _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
+    _contact_cache["fetched_at"] = datetime.utcnow()
+
+
+# ── vCard parsing ──
+
+def _vunesc(value: str) -> str:
+    """Reverse _vesc() — turn escaped vCard text back into the raw value.
+    Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
+    if not value:
+        return value
+    out = []
+    i = 0
+    while i < len(value):
+        ch = value[i]
+        if ch == "\\" and i + 1 < len(value):
+            nxt = value[i + 1]
+            if nxt in ("n", "N"):
+                out.append("\n")
+            elif nxt in (",", ";", "\\"):
+                out.append(nxt)
+            else:
+                out.append(nxt)
+            i += 2
+        else:
+            out.append(ch)
+            i += 1
+    return "".join(out)
+
+
+def _parse_vcards(text: str) -> List[Dict]:
+    """Parse a stream of vCards into dicts with name, email, phone."""
+    # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
+    # space or tab is a continuation of the previous logical line. Real
+    # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
+    # PHOTO lines, and splitting on raw newlines without unfolding dropped the
+    # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
+    # email/name.
+    text = re.sub(r"\r\n[ \t]", "", text or "")
+    text = re.sub(r"\n[ \t]", "", text)
+    contacts = []
+    for block in re.split(r"BEGIN:VCARD", text):
+        if not block.strip():
+            continue
+        contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
+        for line in block.split("\n"):
+            line = line.strip()
+            # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
+            # that Apple Contacts / iCloud / many CardDAV servers emit by
+            # default — without this the property-name checks below miss those
+            # lines and silently drop the email / phone. The group token only
+            # precedes the property name, so it is safe to strip for matching
+            # and value extraction, and a no-op for non-grouped lines.
+            name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
+            if name_part.startswith("FN:") or name_part.startswith("FN;"):
+                contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
+            elif name_part.startswith("EMAIL"):
+                # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
+                if ":" in name_part:
+                    email_addr = _vunesc(name_part.split(":", 1)[1])
+                    if email_addr and email_addr not in contact["emails"]:
+                        contact["emails"].append(email_addr)
+            elif name_part.startswith("TEL"):
+                if ":" in name_part:
+                    phone = _vunesc(name_part.split(":", 1)[1])
+                    if phone and phone not in contact["phones"]:
+                        contact["phones"].append(phone)
+            elif name_part.startswith("ADR"):
+                # vCard ADR is 7 semicolon-separated components:
+                # post-office-box;extended-address;street;locality;region;postal-code;country.
+                # Recover a human-readable string by joining non-empty
+                # components with ", ".
+                if ":" in name_part:
+                    raw = name_part.split(":", 1)[1]
+                    parts = [_vunesc(p).strip() for p in raw.split(";")]
+                    contact["address"] = ", ".join(p for p in parts if p)
+            elif name_part.startswith("UID:"):
+                contact["uid"] = _vunesc(name_part[4:])
+        if contact["name"] or contact["emails"]:
+            contacts.append(contact)
+    return contacts
+
+
+def _vesc(value: str) -> str:
+    """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
+    semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
+    or any value containing a newline produces a malformed vCard (broken
+    N/FN fields) or could inject arbitrary properties."""
+    return (
+        (value or "")
+        .replace("\\", "\\\\")
+        .replace("\n", "\\n")
+        .replace("\r", "")
+        .replace(",", "\\,")
+        .replace(";", "\\;")
+    )
+
+
+def _build_vcard(name: str, email: str, uid: Optional[str] = None,
+                 emails: Optional[List[str]] = None,
+                 phones: Optional[List[str]] = None,
+                 address: Optional[str] = None) -> str:
+    """Build a vCard. Accepts either a single `email` (legacy callers) or
+    full `emails`/`phones` lists (edit path). The first email is marked
+    PREF=1. All values are RFC-6350-escaped."""
+    if not uid:
+        uid = str(uuid.uuid4())
+    # Normalize email lists — `email` arg is a convenience for single-email
+    # creation; `emails` (if given) is authoritative.
+    email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
+    phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
+    # Try to split name into first/last
+    parts = name.strip().split()
+    if len(parts) >= 2:
+        first = parts[0]
+        last = " ".join(parts[1:])
+    else:
+        first = name
+        last = ""
+    # N field is structured (5 components separated by ';') — escape each
+    # component individually so a comma in the name doesn't split it.
+    n_field = f"{_vesc(last)};{_vesc(first)};;;"
+    lines = [
+        "BEGIN:VCARD",
+        "VERSION:4.0",
+        f"UID:{_vesc(uid)}",
+        f"FN:{_vesc(name)}",
+        f"N:{n_field}",
+    ]
+    for i, em in enumerate(email_list):
+        # First email is the preferred one.
+        lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
+    for ph in phone_list:
+        lines.append(f"TEL:{_vesc(ph)}")
+    # Address: stuff the whole human-readable string into the street
+    # component of ADR. vCard ADR has 7 semicolon-separated components:
+    # post-office-box;extended-address;street;locality;region;postal-code;country.
+    addr = (address or "").strip()
+    if addr:
+        lines.append(f"ADR:;;{_vesc(addr)};;;;")
+    lines.append("END:VCARD")
+    return "\r\n".join(lines) + "\r\n"
+
+
+# ── In-memory cache ──
+
+_contact_cache = {"contacts": [], "fetched_at": None}
+
+
+def _abs_url(href: str) -> str:
+    """Combine a multistatus <href> (an absolute path like
+    /user/contacts/x.vcf) with the configured CardDAV server origin so we
+    get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
+    for the configured origin; a cross-origin href is treated as a path on the
+    configured server so a malicious CardDAV response cannot redirect later
+    writes/deletes to cloud metadata or another host."""
+    cfg = _get_carddav_config()
+    base = _carddav_base_url(cfg)
+    base_p = urlparse(base)
+    joined = urljoin(base.rstrip("/") + "/", href or "")
+    joined_p = urlparse(joined)
+    if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
+        joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
+    return _validate_carddav_url(joined)
+
+
+# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
+# alongside the resource href. Lets us map each contact's UID to the real
+# server resource path (which is NOT always <uid>.vcf for contacts created
+# by other clients).
+_ADDRESSBOOK_QUERY = (
+    '<?xml version="1.0" encoding="utf-8"?>'
+    '<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
+    '<D:prop><D:getetag/><C:address-data/></D:prop>'
+    '<C:filter/>'
+    '</C:addressbook-query>'
+)
+
+
+def _fetch_via_report(cfg, auth):
+    """Try a CardDAV REPORT addressbook-query — returns contacts WITH an
+    `href` field, or None if the server doesn't support it / errors."""
+    from defusedxml import ElementTree as ET
+    try:
+        r = httpx.request(
+            "REPORT", cfg["url"],
+            content=_ADDRESSBOOK_QUERY.encode("utf-8"),
+            headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
+            auth=auth, timeout=10,
+        )
+        if r.status_code not in (207, 200):
+            return None
+        root = ET.fromstring(r.text)
+        ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
+        out = []
+        for resp in root.findall("D:response", ns):
+            href_el = resp.find("D:href", ns)
+            data_el = resp.find(".//C:address-data", ns)
+            if href_el is None or data_el is None or not (data_el.text or "").strip():
+                continue
+            parsed = _parse_vcards(data_el.text)
+            if not parsed:
+                continue
+            c = parsed[0]
+            c["href"] = href_el.text.strip()
+            out.append(c)
+        # If the REPORT parsed to ZERO contacts, don't trust it — some
+        # CardDAV servers treat an empty <filter/> as "match nothing" and
+        # return a valid-but-empty 207. Return None so the caller falls
+        # back to the plain GET (which lists everything). A genuinely empty
+        # address book just costs one extra GET that also returns nothing.
+        if not out:
+            return None
+        return out
+    except Exception as e:
+        logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
+        return None
+
+
+def _fetch_contacts(force=False):
+    """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
+    if not force and _contact_cache["fetched_at"]:
+        age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
+        if age < 60:
+            return _contact_cache["contacts"]
+
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        _contact_cache["contacts"] = contacts
+        _contact_cache["fetched_at"] = datetime.utcnow()
+        return contacts
+
+    try:
+        cfg["url"] = _carddav_base_url(cfg)
+        auth = None
+        if cfg["username"]:
+            auth = (cfg["username"], cfg["password"])
+        # Preferred path: REPORT gives us hrefs for reliable edit/delete.
+        contacts = _fetch_via_report(cfg, auth)
+        if contacts is None:
+            # Fallback: plain GET, concatenated vCards, no hrefs.
+            r = httpx.get(cfg["url"], auth=auth, timeout=10)
+            if r.status_code != 200:
+                logger.warning(f"CardDAV returned {r.status_code}")
+                return _contact_cache["contacts"]
+            contacts = _parse_vcards(r.text)
+        _contact_cache["contacts"] = contacts
+        _contact_cache["fetched_at"] = datetime.utcnow()
+        return contacts
+    except Exception as e:
+        logger.error(f"Failed to fetch contacts: {e}")
+        return _contact_cache["contacts"]
+
+
+def _resolve_resource_url(uid: str) -> str:
+    """Map a contact UID to its real CardDAV resource URL. Uses the href
+    captured during fetch when available (handles contacts whose filename
+    != UID); falls back to the <uid>.vcf guess for app-created contacts or
+    when no href is known."""
+    def _lookup():
+        for c in _contact_cache.get("contacts", []):
+            if c.get("uid") == uid and c.get("href"):
+                return _abs_url(c["href"])
+        return None
+    found = _lookup()
+    if found:
+        return found
+    # Not in cache (or no href) — refresh once and retry before guessing.
+    try:
+        _fetch_contacts(force=True)
+    except Exception:
+        pass
+    return _lookup() or _vcard_url(uid)
+
+
+def _create_contact(name: str, email: str, address: str = "") -> bool:
+    """Add a new contact via CardDAV or local contacts."""
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        email_l = (email or "").strip().lower()
+        for c in contacts:
+            if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
+                return True
+        contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address}))
+        _save_local_contacts(contacts)
+        return True
+
+    contact_uid = str(uuid.uuid4())
+    vcard = _build_vcard(name, email, contact_uid, address=address)
+    try:
+        url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
+        auth = None
+        if cfg["username"]:
+            auth = (cfg["username"], cfg["password"])
+        r = httpx.put(
+            url,
+            data=vcard.encode("utf-8"),
+            headers={"Content-Type": "text/vcard; charset=utf-8"},
+            auth=auth,
+            timeout=10,
+        )
+        if r.status_code in (200, 201, 204):
+            # Invalidate cache
+            _contact_cache["fetched_at"] = None
+            return True
+        logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to create contact: {e}")
+        return False
+
+
+def _vcard_url(uid: str) -> str:
+    """The CardDAV resource URL for a given contact UID. The uid is URL-
+    encoded so a value containing '/', '..' or other path chars can't
+    escape the collection and target an arbitrary CardDAV resource."""
+    from urllib.parse import quote
+    cfg = _get_carddav_config()
+    return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
+
+
+def _import_vcards(text: str) -> Dict:
+    """Import a (possibly multi-card) .vcf blob. Each card is PUT to the
+    CardDAV server PRESERVING its full original content (ADR/ORG/photo/
+    etc.) — we don't rebuild it, just ensure it has VERSION + UID and
+    normalize line endings. Returns {imported, failed, total}."""
+    from urllib.parse import quote
+    cfg = _get_carddav_config()
+    if not cfg.get("url"):
+        parsed = _parse_vcards(text)
+        contacts = _load_local_contacts()
+        existing = {
+            e.lower()
+            for c in contacts
+            for e in (c.get("emails") or [])
+            if e
+        }
+        imported = 0
+        for c in parsed:
+            emails = [e for e in (c.get("emails") or []) if e]
+            if emails and any(e.lower() in existing for e in emails):
+                continue
+            contacts.append(_normalize_contact(c))
+            for e in emails:
+                existing.add(e.lower())
+            imported += 1
+        if imported:
+            _save_local_contacts(contacts)
+        return {"imported": imported, "failed": 0, "total": len(parsed)}
+    try:
+        base_url = _carddav_base_url(cfg)
+    except ValueError as e:
+        logger.warning("CardDAV import URL rejected: %s", e)
+        return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
+    auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
+    # Split into individual cards. re.split drops the BEGIN line, so we
+    # re-add it. Normalize CRLF.
+    raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
+    blocks = []
+    for chunk in raw.split("BEGIN:VCARD"):
+        chunk = chunk.strip()
+        if not chunk:
+            continue
+        # Trim anything after END:VCARD (defensive).
+        end = chunk.upper().find("END:VCARD")
+        body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
+        blocks.append("BEGIN:VCARD\n" + body)
+    imported = 0
+    failed = 0
+    for block in blocks:
+        # Extract or assign a UID.
+        m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
+        uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
+        if not m:
+            # Inject a UID right after the VERSION line (or after BEGIN).
+            if re.search(r"^VERSION:", block, re.MULTILINE):
+                block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
+            else:
+                block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
+        elif not re.search(r"^VERSION:", block, re.MULTILINE):
+            block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
+        vcard = block.replace("\n", "\r\n") + "\r\n"
+        url = base_url + "/" + quote(uid, safe="") + ".vcf"
+        try:
+            r = httpx.put(
+                url, data=vcard.encode("utf-8"),
+                headers={"Content-Type": "text/vcard; charset=utf-8"},
+                auth=auth, timeout=15,
+            )
+            if r.status_code in (200, 201, 204):
+                imported += 1
+            else:
+                failed += 1
+                logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
+        except Exception as e:
+            failed += 1
+            logger.error(f"Import PUT {uid} failed: {e}")
+    if imported:
+        _contact_cache["fetched_at"] = None
+    return {"imported": imported, "failed": failed, "total": len(blocks)}
+
+
+def _import_csv_contacts(text: str) -> Dict:
+    """Import contacts from CSV. Supports common headers:
+    name/full_name/display_name, email/email_address/e-mail, phone/tel.
+    Falls back to first columns as name,email,phone when no headers exist."""
+    raw = (text or "").strip()
+    if not raw:
+        return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
+
+    try:
+        sample = raw[:2048]
+        dialect = csv.Sniffer().sniff(sample)
+    except Exception:
+        dialect = csv.excel
+
+    stream = io.StringIO(raw)
+    try:
+        has_header = csv.Sniffer().has_header(raw[:2048])
+    except Exception:
+        has_header = True
+
+    rows = []
+    if has_header:
+        reader = csv.DictReader(stream, dialect=dialect)
+        for row in reader:
+            lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
+            name = (
+                lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
+                or lowered.get("display name") or lowered.get("display_name")
+                or lowered.get("fn") or ""
+            )
+            email = (
+                lowered.get("email") or lowered.get("email address")
+                or lowered.get("email_address") or lowered.get("e-mail")
+                or lowered.get("mail") or ""
+            )
+            phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
+            rows.append((name, email, phone))
+    else:
+        stream.seek(0)
+        reader = csv.reader(stream, dialect=dialect)
+        for row in reader:
+            cols = [(c or "").strip() for c in row]
+            if not any(cols):
+                continue
+            rows.append((
+                cols[0] if len(cols) > 0 else "",
+                cols[1] if len(cols) > 1 else "",
+                cols[2] if len(cols) > 2 else "",
+            ))
+
+    imported = 0
+    failed = 0
+    total = 0
+    existing_emails = {
+        e.lower()
+        for c in _fetch_contacts()
+        for e in (c.get("emails") or [])
+        if e
+    }
+    for name, email, phone in rows:
+        email = (email or "").strip()
+        name = (name or "").strip() or (email.split("@")[0] if email else "")
+        if not email:
+            continue
+        total += 1
+        if email.lower() in existing_emails:
+            continue
+        ok = _create_contact(name, email)
+        if ok:
+            imported += 1
+            existing_emails.add(email.lower())
+            # If the CSV had a phone number, rewrite the just-created row
+            # through the richer update path so phone lands in CardDAV too.
+            if phone:
+                try:
+                    contacts = _fetch_contacts(force=True)
+                    created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
+                    if created and created.get("uid"):
+                        _update_contact(created["uid"], name, [email], [phone])
+                except Exception:
+                    pass
+        else:
+            failed += 1
+
+    if imported:
+        _contact_cache["fetched_at"] = None
+    return {"imported": imported, "failed": failed, "total": total}
+
+
+def _contacts_to_vcf(contacts: List[Dict]) -> str:
+    return "".join(
+        _build_vcard(
+            c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
+            "",
+            uid=c.get("uid") or str(uuid.uuid4()),
+            emails=c.get("emails") or [],
+            phones=c.get("phones") or [],
+        )
+        for c in contacts
+    )
+
+
+def _contacts_to_csv(contacts: List[Dict]) -> str:
+    out = io.StringIO()
+    writer = csv.writer(out)
+    writer.writerow(["name", "email", "phone"])
+    for c in contacts:
+        emails = c.get("emails") or [""]
+        phones = c.get("phones") or [""]
+        max_len = max(len(emails), len(phones), 1)
+        for i in range(max_len):
+            writer.writerow([
+                c.get("name") or "",
+                emails[i] if i < len(emails) else "",
+                phones[i] if i < len(phones) else "",
+            ])
+    return out.getvalue()
+
+
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
+    """Rewrite an existing contact via CardDAV or local contacts."""
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        found = False
+        out = []
+        for c in contacts:
+            if c.get("uid") == uid:
+                # Preserve existing address when caller passes "" (only
+                # updating name/emails/phones, not touching address).
+                addr = address if address else c.get("address", "")
+                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
+                found = True
+            else:
+                out.append(c)
+        if not found:
+            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
+        _save_local_contacts(out)
+        return True
+
+    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
+    # Use the real resource href (handles externally-created contacts whose
+    # filename != UID); falls back to the <uid>.vcf guess.
+    try:
+        url = _resolve_resource_url(uid)
+        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
+        r = httpx.put(
+            url,
+            data=vcard.encode("utf-8"),
+            headers={"Content-Type": "text/vcard; charset=utf-8"},
+            auth=auth,
+            timeout=10,
+        )
+        if r.status_code in (200, 201, 204):
+            _contact_cache["fetched_at"] = None
+            return True
+        logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to update contact: {e}")
+        return False
+
+
+def _delete_contact(uid: str) -> bool:
+    """Delete a contact via CardDAV or local contacts."""
+    cfg = _get_carddav_config()
+    if not _carddav_configured(cfg):
+        contacts = _load_local_contacts()
+        remaining = [c for c in contacts if c.get("uid") != uid]
+        _save_local_contacts(remaining)
+        return True
+
+    try:
+        url = _resolve_resource_url(uid)
+        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
+        r = httpx.delete(url, auth=auth, timeout=10)
+        if r.status_code in (200, 204, 404):
+            # Invalidate cache so the next fetch sees the server truth.
+            _contact_cache["fetched_at"] = None
+            # Verify: force a fresh fetch and check the UID is actually gone.
+            # A 404 on the guessed URL ({uid}.vcf) can mean the contact
+            # lives at a different resource URL — the DELETE missed it but
+            # we'd silently report success. This check catches that.
+            fresh = _fetch_contacts(force=True)
+            still_there = any(c.get("uid") == uid for c in fresh)
+            if still_there:
+                logger.warning(
+                    f"CardDAV DELETE reported success for {uid} "
+                    f"but UID still present after re-fetch — "
+                    f"resource URL may differ from {redact_url(url)}"
+                )
+                return False
+            if r.status_code == 404:
+                logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
+            return True
+        logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
+        return False
+    except Exception as e:
+        logger.error(f"Failed to delete contact: {e}")
+        return False
+
+
+# ── Routes ──
+
+def setup_contacts_routes():
+    router = APIRouter(prefix="/api/contacts", tags=["contacts"])
+
+    @router.get("/list")
+    async def list_contacts(_admin: str = Depends(require_admin)):
+        """List all contacts."""
+        contacts = _fetch_contacts()
+        return {"contacts": contacts, "count": len(contacts)}
+
+    @router.get("/search")
+    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
+        """Search contacts by name or email. Returns up to 10 matches."""
+        contacts = _fetch_contacts()
+        if not q:
+            return {"results": []}
+        q_lower = q.lower()
+        results = []
+        for c in contacts:
+            if q_lower in c["name"].lower():
+                results.append(c)
+                continue
+            for em in c["emails"]:
+                if q_lower in em.lower():
+                    results.append(c)
+                    break
+        return {"results": results[:10]}
+
+    @router.post("/add")
+    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
+        """Add a new contact."""
+        name = (data.get("name") or "").strip()
+        email = (data.get("email") or "").strip()
+        phone = (data.get("phone") or "").strip()
+        address = (data.get("address") or "").strip()
+        if not email:
+            return {"success": False, "error": "Email required"}
+        # Check if already exists by email
+        if email:
+            contacts = _fetch_contacts()
+            for c in contacts:
+                if email.lower() in [e.lower() for e in c["emails"]]:
+                    return {"success": True, "message": "Already exists", "contact": c}
+        if not name:
+            name = email.split("@")[0]
+        create_params = inspect.signature(_create_contact).parameters
+        if len(create_params) >= 3:
+            ok = _create_contact(name, email, address)
+        else:
+            ok = _create_contact(name, email)
+        # If a phone was provided, do an immediate update to thread it
+        # through (the simple _create_contact signature only takes name +
+        # email + address; phones happen via update).
+        if ok and phone:
+            try:
+                fresh = _fetch_contacts(force=True)
+                created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
+                if created:
+                    _update_contact(
+                        created["uid"], name,
+                        created.get("emails", []),
+                        [phone],
+                        address,
+                    )
+            except Exception:
+                pass
+        return {"success": ok}
+
+    @router.post("/import")
+    async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
+        """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
+        # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
+        # in the JSON body) would otherwise reach .strip() and 500 with an
+        # AttributeError instead of degrading to a clean "no data" response.
+        text = str(data.get("vcf") or data.get("text") or "")
+        csv_text = str(data.get("csv") or "")
+        if text.strip():
+            if "BEGIN:VCARD" not in text.upper():
+                return {"success": False, "error": "No vCard data found"}
+            result = _import_vcards(text)
+        elif csv_text.strip():
+            result = _import_csv_contacts(csv_text)
+        else:
+            return {"success": False, "error": "No contact data found"}
+        result["success"] = result.get("imported", 0) > 0
+        return result
+
+    @router.get("/export")
+    async def export_contacts(
+        format: str = Query("vcf", pattern="^(vcf|csv)$"),
+        _admin: str = Depends(require_admin),
+    ):
+        """Export all contacts as vCard or CSV."""
+        contacts = _fetch_contacts(force=True)
+        if format == "csv":
+            content = _contacts_to_csv(contacts)
+            media_type = "text/csv; charset=utf-8"
+            filename = "odysseus-contacts.csv"
+        else:
+            content = _contacts_to_vcf(contacts)
+            media_type = "text/vcard; charset=utf-8"
+            filename = "odysseus-contacts.vcf"
+        return Response(
+            content=content,
+            media_type=media_type,
+            headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+        )
+
+    @router.get("/config")
+    async def get_config(_admin: str = Depends(require_admin)):
+        cfg = _get_carddav_config()
+        # Mask password
+        if cfg["password"]:
+            cfg["password"] = "***"
+        return cfg
+
+    @router.put("/config")
+    async def update_config(data: dict, _admin: str = Depends(require_admin)):
+        settings = _load_settings()
+        for key in ("carddav_url", "carddav_username", "carddav_password"):
+            if key in data:
+                if key == "carddav_url" and str(data[key] or "").strip():
+                    try:
+                        settings[key] = _validate_carddav_url(data[key])
+                    except ValueError as e:
+                        raise HTTPException(400, str(e))
+                else:
+                    value = data[key]
+                    if key == "carddav_password" and value:
+                        from src.secret_storage import encrypt
+                        value = encrypt(value)
+                    settings[key] = value
+        _save_settings(settings)
+        # Force re-fetch
+        _contact_cache["fetched_at"] = None
+        return {"success": True}
+
+    @router.delete("/clear")
+    async def clear_contacts(_admin: str = Depends(require_admin)):
+        """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
+        _save_local_contacts([])
+        return {"success": True}
+
+    # NOTE: the /{uid} routes are declared LAST so the literal paths above
+    # (/list, /search, /add, /config) win — otherwise PUT /config would
+    # match PUT /{uid} with uid="config".
+    @router.put("/{uid}")
+    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
+        """Edit an existing contact — name / emails / phones / address."""
+        name = (data.get("name") or "").strip()
+        emails = data.get("emails")
+        phones = data.get("phones")
+        if emails is None and data.get("email"):
+            emails = [data["email"]]
+        emails = [e.strip() for e in (emails or []) if e and e.strip()]
+        phones = [p.strip() for p in (phones or []) if p and p.strip()]
+        address = (data.get("address") or "").strip()
+        if not name and not emails and not address:
+            return {"success": False, "error": "Name, email, or address required"}
+        if not name and emails:
+            name = emails[0].split("@")[0]
+        ok = _update_contact(uid, name, emails, phones, address)
+        return {"success": ok}
+
+    @router.delete("/{uid}")
+    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
+        """Delete a contact by UID."""
+        if not uid:
+            return {"success": False, "error": "UID required"}
+        ok = _delete_contact(uid)
+        return {"success": ok}
+
+    return router

routes/cookbook_helpers.py

@@ -439,30 +439,15 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "                if f.is_file(): nf += 1; sz += f.stat().st_size",
         "                if f.name.endswith('.incomplete'): ic = True",
         "        snap = os.path.join(cache, d, 'snapshots')",
-        "        def snapshot_size():",
+        "        # Windows HF cache stores files directly in snapshots/; blobs/ may be empty.",
-        "            total, count, incomplete = 0, 0, False",
+        "        # Fallback: scan snapshots for real files when blobs yielded nothing.",
-        "            seen_real = set()",
+        "        if sz == 0 and os.path.isdir(snap):",
         "            for sd in os.listdir(snap):",
         "                sf = os.path.join(snap, sd)",
         "                if not os.path.isdir(sf): continue",
-        "                for root, dirs, fns in safe_walk(sf):",
+        "                for f in os.scandir(sf):",
-        "                    for fn in fns:",
+        "                    if f.is_file(): nf += 1; sz += f.stat().st_size",
-        "                        fp = os.path.join(root, fn)",
+        "                    if f.name.endswith('.incomplete'): ic = True",
-        "                        if fn.endswith('.incomplete'): incomplete = True",
-        "                        try:",
-        "                            real = os.path.realpath(fp)",
-        "                            if real in seen_real: continue",
-        "                            seen_real.add(real)",
-        "                            total += os.path.getsize(real)",
-        "                            count += 1",
-        "                        except Exception:",
-        "                            pass",
-        "            return total, count, incomplete",
-        "        # Some HF caches (macOS/MLX/Xet-style) keep blobs elsewhere or expose",
-        "        # snapshot symlinks only. Size snapshots too when blob accounting is empty.",
-        "        if sz == 0 and os.path.isdir(snap):",
-        "            sz2, nf2, ic2 = snapshot_size()",
-        "            sz, nf, ic = sz2, nf2, ic or ic2",
         "        is_diffusion = False; gguf_files = []",
         "        if os.path.isdir(snap):",
         "            for sd in os.listdir(snap):",
@@ -486,18 +471,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "    add('/app/.cache/huggingface/hub')",
         f"    add({add_hf_cache!r})" if add_hf_cache else "",
         "    return candidates",
-        "def normalize_model_dir(p):",
-        "    p = os.path.expanduser((p or '').strip())",
-        "    if not p: return p",
-        "    if os.path.isdir(p) or os.path.isabs(p): return p",
-        "    # Users often paste Linux absolute paths without the leading slash.",
-        "    # Treat home/<user>/... as /home/<user>/... so remote scans work.",
-        "    if p.startswith(('home/', 'mnt/', 'media/', 'data/', 'opt/', 'srv/', 'var/')):",
-        "        prefixed = '/' + p",
-        "        if os.path.isdir(prefixed): return prefixed",
-        "    return p",
         "def scan_dir(p):",
-        "    p = normalize_model_dir(p)",
         "    if not os.path.isdir(p) or not safe_path(p): return",
         "    for d in sorted(os.listdir(p)):",
         "        if d.startswith('.'): continue",
@@ -567,7 +541,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "scan_ollama()",
     ]
     for model_dir in model_dirs or []:
-        lines.append(f"scan_dir({model_dir!r})")
+        lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))")
     lines.append("print(json.dumps(models))")
     return "\n".join(lines) + "\n"
 
@@ -584,18 +558,6 @@ def _bash_squote(v: str) -> str:
     return v.replace("'", "'\\''")
 
 
-# Shown by generated runner scripts when the ollama binary is missing on the
-# target host. Must stay free of backticks/$( ) and be emitted single-quoted:
-# an earlier version wrapped the install one-liner in backticks inside a
-# double-quoted echo, which bash executed as command substitution and ran the
-# system-wide installer (including on remote SSH hosts) instead of printing
-# the hint.
-OLLAMA_MISSING_HINT = (
-    "ERROR: Ollama not found on this server. Install it from "
-    "https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh"
-)
-
-
 # Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
 # Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
 _SERVE_CMD_ALLOWLIST = {
@@ -615,16 +577,6 @@ _SERVE_CMD_ALLOWLIST = {
 _GGUF_PRELUDE_RE = re.compile(
     r'^MODEL_FILE=\$\([^\n]*?\)\s*&&\s*\{[^{}]*\}\s*\|\|\s*\{[^{}]*\}\s*&&\s*'
 )
-_SAFE_SUBSHELL_TEXT = r"[^'\n;&|`$()<>]+"
-_SAFE_SUBSHELL_DQ_HOME_PATH = r'"\$HOME/[^"\n;&|`()<>]*"'
-_SAFE_PRINTF_SUBSHELL_RE = re.compile(
-    rf"^\$\(printf[ \t]+%s[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|\$\{{HOME\}}'/{_SAFE_SUBSHELL_TEXT}')\)$"
-)
-_SAFE_FIND_MMPROJ_SUBSHELL_RE = re.compile(
-    rf"^\$\(find[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|{_SAFE_SUBSHELL_DQ_HOME_PATH}|{_SAFE_SUBSHELL_TEXT})"
-    r"[ \t]+-iname[ \t]+'mmproj\*\.gguf'"
-    r"(?:[ \t]+2>/dev/null)?[ \t]*\|[ \t]*sort[ \t]*\|[ \t]*head[ \t]+-1\)$"
-)
 _OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)")
 _OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$")
 _OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
@@ -725,13 +677,6 @@ def _check_serve_binary(seg: str) -> None:
         )
 
 
-def _is_safe_serve_subshell(subshell: str) -> bool:
-    return bool(
-        _SAFE_PRINTF_SUBSHELL_RE.fullmatch(subshell)
-        or _SAFE_FIND_MMPROJ_SUBSHELL_RE.fullmatch(subshell)
-    )
-
-
 def _validate_serve_cmd(v: str | None) -> str | None:
     """Reject serve commands that aren't in the allowlist or contain shell metachars.
 
@@ -763,15 +708,15 @@ def _validate_serve_cmd(v: str | None) -> str | None:
             _check_serve_binary(part.strip())
         return v
 
-    # Otherwise: a single invocation — no shell metacharacters allowed. Replace
+    # Otherwise: a single invocation — no shell metacharacters allowed.
-    # only the exact command substitutions emitted by the Cookbook UI:
+    # Temporarily replace safe $(printf %s ...) expressions with a placeholder
-    # $(printf %s 'safe-path') and the mmproj lookup
+    # to avoid triggering the metacharacter/command-injection checks.
-    # $(find <path> -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1).
+    cleaned_v = v
-    def _replace_safe_subshell(match: re.Match[str]) -> str:
+    printf_matches = list(re.finditer(r"\$\(\s*printf\s+%s\s+([^\n()]*?)\)", v))
-        subshell = match.group(0)
+    for match in printf_matches:
-        return "/placeholder/safe/path" if _is_safe_serve_subshell(subshell) else subshell
+        inner = match.group(1)
-
+        if not any(c in inner for c in (";", "&&", "||", "$(", "`")):
-    cleaned_v = re.sub(r"\$\([^()]*\)", _replace_safe_subshell, v)
+            cleaned_v = cleaned_v.replace(match.group(0), "/placeholder/safe/path.gguf")
 
     # (`$(` was the original intent; bare `$` is fine for shell-safe paths.)
     if any(c in cleaned_v for c in (";", "&&", "||", "$(")):
@@ -1299,15 +1244,13 @@ def _diagnose_serve_output(text: str) -> dict | None:
             [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
         ),
         (
-            r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
+            r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
-            r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
+            r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
-            r"Could not load any common_ops library|"
             r"Please ensure sgl_kernel is properly installed",
-            "SGLang native kernel/runtime is missing or mismatched on this server.",
+            "SGLang native dependencies are missing on this server.",
             [
-                {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
                 {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
-                {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
+                {"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
             ],
         ),
         (
@@ -1315,19 +1258,6 @@ def _diagnose_serve_output(text: str) -> dict | None:
             "SGLang is not installed or not in PATH on this server.",
             [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
         ),
-        (
-            r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
-            "MLX LM is not installed on this server.",
-            [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
-        ),
-        (
-            r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
-            "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
-            [
-                {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
-                {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
-            ],
-        ),
         # System build deps come BEFORE the generic llama.cpp catch-all so
         # cmake / build-essential / git missing → a specific OS-package
         # remediation instead of "install llama-cpp-python[server]" (which

routes/document_helpers.py

@@ -29,7 +29,6 @@ class DocumentCreate(BaseModel):
 class DocumentUpdate(BaseModel):
     content: str
     summary: Optional[str] = None
-    force_version: bool = False
 
 class DocumentPatch(BaseModel):
     title: Optional[str] = None

routes/document_routes.py

@@ -54,18 +54,6 @@ def _library_language_for_document(doc: Document) -> str:
     return doc.language or "text"
 
 
-def _email_source_key(content: str) -> tuple[str, str]:
-    """Return the source email identity embedded in an email draft document."""
-    import re
-
-    text = content or ""
-    uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text)
-    folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text)
-    uid = (uid_m.group(1).strip() if uid_m else "")
-    folder = (folder_m.group(1).strip() if folder_m else "INBOX")
-    return uid, folder
-
-
 from routes.document_helpers import (
     DocumentCreate, DocumentUpdate, DocumentPatch,
     _doc_to_dict, _version_to_dict,
@@ -111,62 +99,24 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 # the existing lenient path.
                 session = _get_session_or_404(db, req.session_id, user)
 
+            doc_id = str(uuid.uuid4())
+            ver_id = str(uuid.uuid4())
+
             # If no language was supplied (e.g. cloning a doc whose language
             # was never set), detect it from the content rather than storing
             # NULL — which made the editor fall back to plain text. Defaults
             # to markdown for prose.
             language = req.language
             if not language:
-                from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content
+                from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language
                 language = _sniff_doc_language(req.content)
             else:
-                from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content
+                from src.agent_tools.document_tools import _looks_like_email_document
             if _looks_like_email_document(req.content, req.title):
                 language = "email"
 
             _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
 
-            # Reply drafts are keyed to the source email. If a UI/tool path tries
-            # to create a second draft for the same email in the same chat,
-            # update the existing draft instead so quoted thread history stays
-            # attached to the visible document.
-            if language == "email" and req.session_id:
-                source_uid, source_folder = _email_source_key(req.content)
-                if source_uid:
-                    candidates = (
-                        db.query(Document)
-                        .filter(Document.session_id == req.session_id)
-                        .filter(Document.is_active == True)
-                        .filter(Document.language == "email")
-                        .order_by(Document.updated_at.desc())
-                        .limit(25)
-                        .all()
-                    )
-                    for existing in candidates:
-                        old_uid, old_folder = _email_source_key(existing.current_content or "")
-                        if old_uid != source_uid or old_folder != source_folder:
-                            continue
-                        merged = _coerce_email_document_content(existing.current_content or "", req.content)
-                        if existing.current_content != merged:
-                            new_ver = (existing.version_count or 1) + 1
-                            existing.current_content = merged
-                            existing.title = req.title or existing.title
-                            existing.version_count = new_ver
-                            db.add(DocumentVersion(
-                                id=str(uuid.uuid4()),
-                                document_id=existing.id,
-                                version_number=new_ver,
-                                content=merged,
-                                summary="Updated existing email draft",
-                                source="user",
-                            ))
-                            db.commit()
-                            db.refresh(existing)
-                        return _doc_to_dict(existing)
-
-            doc_id = str(uuid.uuid4())
-            ver_id = str(uuid.uuid4())
-
             doc = Document(
                 id=doc_id,
                 session_id=req.session_id,
@@ -620,23 +570,11 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                 raise HTTPException(404, "Document not found")
             _verify_doc_owner(db, doc, user)
 
-            incoming_content = req.content
+            # Skip if content is identical
-            from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document
+            if doc.current_content == req.content:
-            is_email_doc = (
-                (doc.language or "").lower() == "email"
-                or _looks_like_email_document(doc.current_content or "", doc.title or "")
-                or _looks_like_email_document(req.content or "", doc.title or "")
-            )
-            if is_email_doc:
-                incoming_content = _coerce_email_document_content(doc.current_content or "", req.content)
-                doc.language = "email"
-
-            # Skip if content is identical unless the caller explicitly wants
-            # a checkpoint version from the current editor state.
-            if doc.current_content == incoming_content and not req.force_version:
                 return _doc_to_dict(doc)
 
-            _assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler)
+            _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
 
             # Check if we can coalesce with the latest version
             latest_ver = db.query(DocumentVersion).filter(
@@ -645,14 +583,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
 
             now = datetime.now(timezone.utc)
             coalesced = False
-            if latest_ver and latest_ver.source == "user" and not req.force_version:
+            if latest_ver and latest_ver.source == "user":
                 ver_time = latest_ver.created_at
                 if ver_time.tzinfo is None:
                     ver_time = ver_time.replace(tzinfo=timezone.utc)
                 age = (now - ver_time).total_seconds()
                 if age < VERSION_COALESCE_SECONDS:
                     # Update the existing version in-place
-                    latest_ver.content = incoming_content
+                    latest_ver.content = req.content
                     latest_ver.created_at = now
                     if req.summary:
                         latest_ver.summary = req.summary
@@ -664,14 +602,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                     id=str(uuid.uuid4()),
                     document_id=doc_id,
                     version_number=new_ver,
-                    content=incoming_content,
+                    content=req.content,
                     summary=req.summary or "Manual edit",
                     source="user",
                 )
                 doc.version_count = new_ver
                 db.add(ver)
 
-            doc.current_content = incoming_content
+            doc.current_content = req.content
             db.commit()
             db.refresh(doc)
             return _doc_to_dict(doc)
@@ -861,26 +799,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
             from src.document_actions import _JUNK_TITLES
 
             to_delete = []
-            now = datetime.now(timezone.utc)
             for doc in docs:
-                created = doc.created_at
-                if created and created.tzinfo is None:
-                    created = created.replace(tzinfo=timezone.utc)
-
-                # Skip freshly created documents to avoid deleting them while the user is actively editing
-                if created and (now - created).total_seconds() < 900:  # 15 minutes
-                    continue
-
                 content = (doc.current_content or "").strip()
                 title_raw = (doc.title or "").strip()
                 title = title_raw.lower()
-                is_fresh_empty = (
-                    not content
-                    and created is not None
-                    and (now - created).total_seconds() < 1800
-                )
-                if is_fresh_empty:
-                    continue
 
                 # Strip markdown noise to get a "real" character count
                 stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE)
@@ -915,6 +837,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                     to_delete.append(doc); deleted += 1; continue
                 if title in _JUNK_TITLES:
                     to_delete.append(doc); deleted += 1; continue
+                if real_len < 30:
+                    to_delete.append(doc); deleted += 1; continue
+                if "\n" not in content and real_len < 50:
+                    to_delete.append(doc); deleted += 1; continue
 
                 # Fix empty or placeholder titles on survivors
                 if not title_raw or title_raw == "Untitled":

routes/email_helpers.py

@@ -349,7 +349,7 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
             row = db.query(_EA).filter(_EA.id == account_id).first()
             if row is None:
                 raise HTTPException(404, "Account not found")
-            if not _account_visible_to_owner(row, owner):
+            if row.owner and row.owner != owner:
                 # Treat as 404 (not 403) so we don't leak existence.
                 raise HTTPException(404, "Account not found")
         finally:
@@ -362,26 +362,6 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
         logger.error(f"Account-owner check failed: {e}")
         raise HTTPException(503, "Account check failed")
 
-
-def _account_visible_to_owner(row, owner: str) -> bool:
-    """Whether an authenticated `owner` may act on this EmailAccount row.
-
-    Mirrors the SQL predicate in `_get_email_config`'s
-    `_owner_or_matching_legacy_account`: a caller sees an account they own, or a
-    legacy owner-less account (owner NULL/"") only when its own mailbox
-    (`imap_user` / `from_address`) is the caller's. `email_accounts` is the one
-    owner-scoped table deliberately left out of the legacy-owner migration
-    backfill, so ownerless rows persist on multi-user deploys — making this the
-    gate that keeps one tenant off another's imported mailbox and its decrypted
-    IMAP/SMTP credentials."""
-    row_owner = getattr(row, "owner", None) or ""
-    if row_owner:
-        return row_owner == owner
-    return owner in {
-        getattr(row, "imap_user", None) or "",
-        getattr(row, "from_address", None) or "",
-    }
-
 def _q(name: str) -> str:
     """Quote an IMAP mailbox name. Defensive: escapes `\\` and `"` and wraps
     in double quotes so user-supplied folder names with spaces or quotes can't
@@ -444,19 +424,12 @@ SCHEDULED_DB = Path(SCHEDULED_EMAILS_DB)
 OWNER_SCOPED_EMAIL_CACHE_TABLES = {
     "email_summaries",
     "email_ai_replies",
-    "email_translations",
     "email_calendar_extractions",
     "email_urgency_alerts",
     "sender_signatures",
 }
 
 
-def email_translation_body_hash(body: str) -> str:
-    import hashlib as _hashlib
-    normalized = (body or "").strip()
-    return _hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest()
-
-
 def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
     owner = (owner or "").strip()
     if owner:
@@ -464,34 +437,14 @@ def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
     return "(owner = '' OR owner IS NULL)", ()
 
 
-def _ensure_owner_scoped_email_cache_table(
+def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, columns: list[str]):
-    conn,
-    table: str,
-    create_sql: str,
-    columns: list[str],
-    pk_columns: list[str] | None = None,
-):
     """Rebuild legacy Message-ID-only cache tables with owner in the PK."""
-    desired_pk_cols = pk_columns or ["message_id", "owner"]
     conn.execute(create_sql)
     try:
         info = conn.execute(f"PRAGMA table_info({table})").fetchall()
         cols = [r[1] for r in info]
         pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
-        for col in columns:
+        if "owner" in cols and pk_cols == ["message_id", "owner"]:
-            if col not in cols:
-                if col == "owner":
-                    conn.execute(f"ALTER TABLE {table} ADD COLUMN owner TEXT DEFAULT ''")
-                elif col in {"event_uids"}:
-                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT '[]'")
-                elif col.startswith("has_") or col.endswith("_created") or col.endswith("_count"):
-                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} INTEGER DEFAULT 0")
-                elif col == "created_at":
-                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT ''")
-                else:
-                    conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT")
-                cols.append(col)
-        if "owner" in cols and pk_cols == desired_pk_cols:
             return
 
         conn.execute(f"ALTER TABLE {table} RENAME TO {table}__old")
@@ -624,25 +577,6 @@ def _init_scheduled_db():
             PRIMARY KEY (message_id, owner)
         )
     """, ["message_id", "owner", "uid", "folder", "reply", "model_used", "created_at"])
-    _ensure_owner_scoped_email_cache_table(conn, "email_translations", """
-        CREATE TABLE IF NOT EXISTS email_translations (
-            body_hash TEXT,
-            owner TEXT DEFAULT '',
-            target_language TEXT DEFAULT 'English',
-            uid TEXT,
-            folder TEXT,
-            subject TEXT,
-            sender TEXT,
-            translation TEXT,
-            same_language INTEGER DEFAULT 0,
-            model_used TEXT,
-            created_at TEXT NOT NULL,
-            PRIMARY KEY (body_hash, owner, target_language)
-        )
-    """, [
-        "body_hash", "owner", "target_language", "uid", "folder", "subject", "sender",
-        "translation", "same_language", "model_used", "created_at",
-    ], ["body_hash", "owner", "target_language"])
     # Email tags / spam classification cache. SECURITY: keyed by
     # (message_id, owner) because Message-IDs are GLOBAL (a newsletter goes
     # to many users with the same Message-ID). Without owner-scoping, a
@@ -652,7 +586,6 @@ def _init_scheduled_db():
         CREATE TABLE IF NOT EXISTS email_tags (
             message_id TEXT,
             owner TEXT DEFAULT '',
-            account_id TEXT DEFAULT '',
             uid TEXT,
             folder TEXT,
             subject TEXT,
@@ -663,7 +596,7 @@ def _init_scheduled_db():
             moved_to TEXT,
             model_used TEXT,
             created_at TEXT NOT NULL,
-            PRIMARY KEY (message_id, owner, account_id)
+            PRIMARY KEY (message_id, owner)
         )
     """)
     # Backfill migration: older installs created the table with
@@ -671,35 +604,28 @@ def _init_scheduled_db():
     # promote it into the PK by rebuild-copy-swap (SQLite can't ALTER PK).
     try:
         _cols = [r[1] for r in conn.execute("PRAGMA table_info(email_tags)")]
-        _pk_cols = [r[1] for r in sorted(conn.execute("PRAGMA table_info(email_tags)").fetchall(), key=lambda row: row[5] or 99) if r[5]]
         if "owner" not in _cols:
+            # Add the column first so reads/writes don't break mid-migration.
             conn.execute("ALTER TABLE email_tags ADD COLUMN owner TEXT DEFAULT ''")
-            _cols.append("owner")
+            # Rebuild with composite PK. Existing rows get owner='' (legacy
-        if "account_id" not in _cols:
+            # single-user); the urgency scanner will overwrite as it
-            conn.execute("ALTER TABLE email_tags ADD COLUMN account_id TEXT DEFAULT ''")
+            # re-classifies. No data loss.
-            _cols.append("account_id")
-        if _pk_cols != ["message_id", "owner", "account_id"]:
-            # Rebuild with account-aware composite PK. Existing rows get
-            # account_id='' and are still readable as legacy fallback rows;
-            # fresh task runs write exact account ids and no longer block each
-            # other when two accounts share a Message-ID.
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS email_tags__new (
                     message_id TEXT,
                     owner TEXT DEFAULT '',
-                    account_id TEXT DEFAULT '',
                     uid TEXT, folder TEXT, subject TEXT, sender TEXT,
                     tags TEXT, spam_verdict INTEGER DEFAULT 0,
                     spam_reason TEXT, moved_to TEXT, model_used TEXT,
                     created_at TEXT NOT NULL,
-                    PRIMARY KEY (message_id, owner, account_id)
+                    PRIMARY KEY (message_id, owner)
                 )
             """)
             conn.execute("""
                 INSERT OR IGNORE INTO email_tags__new
-                  (message_id, owner, account_id, uid, folder, subject, sender, tags,
+                  (message_id, owner, uid, folder, subject, sender, tags,
                    spam_verdict, spam_reason, moved_to, model_used, created_at)
-                SELECT message_id, COALESCE(owner, ''), COALESCE(account_id, ''), uid, folder, subject,
+                SELECT message_id, COALESCE(owner, ''), uid, folder, subject,
                        sender, tags, spam_verdict, spam_reason, moved_to,
                        model_used, created_at
                 FROM email_tags
@@ -715,12 +641,11 @@ def _init_scheduled_db():
             message_id TEXT,
             owner TEXT DEFAULT '',
             uid TEXT,
-            event_uids TEXT DEFAULT '[]',
             events_created INTEGER DEFAULT 0,
             created_at TEXT NOT NULL,
             PRIMARY KEY (message_id, owner)
         )
-    """, ["message_id", "owner", "uid", "event_uids", "events_created", "created_at"])
+    """, ["message_id", "owner", "uid", "events_created", "created_at"])
     _ensure_owner_scoped_email_cache_table(conn, "email_urgency_alerts", """
         CREATE TABLE IF NOT EXISTS email_urgency_alerts (
             message_id TEXT,
@@ -746,64 +671,6 @@ def _init_scheduled_db():
             PRIMARY KEY (owner, account_key, folder, message_key)
         )
     """)
-    conn.execute("""
-        CREATE TABLE IF NOT EXISTS email_message_index (
-            owner TEXT NOT NULL DEFAULT '',
-            account_key TEXT NOT NULL DEFAULT '',
-            folder TEXT NOT NULL,
-            uid TEXT NOT NULL,
-            message_id TEXT,
-            subject TEXT,
-            from_name TEXT,
-            from_address TEXT,
-            to_text TEXT,
-            cc_text TEXT,
-            date_iso TEXT,
-            date_display TEXT,
-            date_epoch REAL DEFAULT 0,
-            size INTEGER DEFAULT 0,
-            flags TEXT DEFAULT '',
-            has_attachments INTEGER DEFAULT 0,
-            updated_at TEXT NOT NULL,
-            PRIMARY KEY (owner, account_key, folder, uid)
-        )
-    """)
-    conn.execute("""
-        CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
-        ON email_message_index(owner, account_key, folder, date_epoch DESC)
-    """)
-    conn.execute("""
-        CREATE INDEX IF NOT EXISTS ix_email_message_index_message_id
-        ON email_message_index(owner, account_key, message_id)
-    """)
-    conn.execute("""
-        CREATE TABLE IF NOT EXISTS email_body_preview_cache (
-            owner TEXT NOT NULL DEFAULT '',
-            account_key TEXT NOT NULL DEFAULT '',
-            folder TEXT NOT NULL,
-            uid TEXT NOT NULL,
-            message_id TEXT,
-            payload_json TEXT NOT NULL,
-            updated_at TEXT NOT NULL,
-            PRIMARY KEY (owner, account_key, folder, uid)
-        )
-    """)
-    conn.execute("""
-        CREATE INDEX IF NOT EXISTS ix_email_body_preview_message_id
-        ON email_body_preview_cache(owner, account_key, message_id)
-    """)
-    conn.execute("""
-        CREATE TABLE IF NOT EXISTS email_attachment_metadata_cache (
-            owner TEXT NOT NULL DEFAULT '',
-            account_key TEXT NOT NULL DEFAULT '',
-            folder TEXT NOT NULL,
-            uid TEXT NOT NULL,
-            message_id TEXT,
-            attachments_json TEXT NOT NULL,
-            updated_at TEXT NOT NULL,
-            PRIMARY KEY (owner, account_key, folder, uid)
-        )
-    """)
     # Boundary cache — LLM-detected sig/quote start positions in the body.
     # Stored as char offsets (-1 = no boundary found). Once cached, the
     # client uses these to fold without ever re-calling the LLM.
@@ -923,13 +790,12 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
         try:
             if account_id:
                 row = db.query(_EA).filter(_EA.id == account_id, _EA.enabled == True).first()  # noqa: E712
-                # If the resolved row isn't visible to this owner, treat as
+                # If the resolved row belongs to a different owner, treat as
                 # not-found rather than silently serving it. This is a defense
                 # in depth — `require_owner` already calls `_assert_owns_account`
                 # for query-param account_ids, but other callers (cookbook
-                # rules, scheduled poller) may not. Ownerless legacy rows are
+                # rules, scheduled poller) may not.
-                # only visible on a mailbox match, same as the fallback below.
+                if row is not None and owner and row.owner and row.owner != owner:
-                if row is not None and owner and not _account_visible_to_owner(row, owner):
                     row = None
             # Fallback path — restrict to this owner's accounts so we don't
             # leak another user's default mailbox to an unconfigured user.
@@ -1294,15 +1160,10 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str
     try:
         c = _imap_connect(account_id, owner=owner)
         c.select(_q(src))
-        # Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy()
+        status, _ = c.copy(uid, _q(dest))
-        # and store() operate on message SEQUENCE NUMBERS, so addressing them
-        # with a UID moved/deleted the wrong message (or silently no-oped when
-        # the UID exceeded the message count). Use the UID commands, matching
-        # the move/delete path in email_routes.py.
-        status, _ = c.uid("COPY", uid, _q(dest))
         if status != "OK":
             return False
-        c.uid("STORE", uid, "+FLAGS", "\\Deleted")
+        c.store(uid, "+FLAGS", "\\Deleted")
         c.expunge()
         return True
     except Exception as e:
@@ -1415,14 +1276,12 @@ def _list_attachments_from_msg(msg):
             except Exception:
                 payload = b""
         size = len(payload) if payload is not None else 0
-        content_id = (part.get("Content-ID") or "").strip().strip("<>")
         attachments.append({
             "index": idx,
             "filename": filename,
             "content_type": ct,
             "size": size,
             "is_inline": "inline" in cd.lower(),
-            "content_id": content_id,
         })
         idx += 1
     return attachments
@@ -1861,10 +1720,6 @@ class SendEmailRequest(BaseModel):
     attachments: Optional[List[str]] = None
     # Which account to send from. None = default account.
     account_id: Optional[str] = None
-    # Source message for replies. When present, /send marks this exact message
-    # answered after successful delivery so it leaves undone/reply-soon views.
-    source_uid: Optional[str] = None
-    source_folder: Optional[str] = None
     # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
     odysseus_kind: Optional[str] = None
     # If true, /send waits for SMTP + Sent append and returns the sent UID.

routes/email_pollers.py

@@ -29,7 +29,7 @@ from datetime import datetime
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
-from src.task_endpoint import resolve_task_candidates, task_llm_call_async
+from src.llm_core import llm_call_async
 
 from routes.email_helpers import (
     _strip_think, _extract_reply, _apply_email_style_mechanics, _load_settings, _save_settings, _get_email_config,
@@ -56,35 +56,6 @@ _CAL_ACTION_ARRAY_RE = re.compile(
 )
 
 
-def _extract_json_array_from_text(text: str):
-    """Return the last valid JSON array embedded in model output, if any."""
-    if not text:
-        return None
-    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip(), flags=re.MULTILINE).strip()
-    decoder = json.JSONDecoder()
-    try:
-        parsed = decoder.decode(cleaned)
-        if isinstance(parsed, list):
-            return parsed
-    except Exception:
-        pass
-
-    # Models often explain themselves and finish with `[]` or `[{"action":...}]`.
-    # Scan every array opener and keep the last complete JSON array, rather than
-    # using a greedy regex that can swallow prose containing square brackets.
-    last = None
-    for idx, ch in enumerate(cleaned):
-        if ch != "[":
-            continue
-        try:
-            parsed, _end = decoder.raw_decode(cleaned[idx:])
-        except Exception:
-            continue
-        if isinstance(parsed, list):
-            last = parsed
-    return last
-
-
 def _owner_for_email_account(account_id: str | None) -> str:
     if not account_id:
         return ""
@@ -117,8 +88,6 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
                                    do_tag: bool = False, do_spam: bool = False,
                                    do_calendar: bool = False,
                                    days_back: int = 1,
-                                   account_id: str | None = None,
-                                   max_process: int | None = None,
                                    progress_cb=None) -> str:
     """One iteration of the email scan. Temporarily flips settings flags
     so the existing background-loop logic runs exactly once for the requested ops."""
@@ -133,12 +102,7 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
     settings["email_auto_calendar"] = bool(do_calendar)
     _save_settings(settings)
     try:
-        return await _auto_summarize_pass(
+        return await _auto_summarize_pass(days_back=days_back, progress_cb=progress_cb)
-            days_back=days_back,
-            account_id=account_id,
-            max_process=max_process,
-            progress_cb=progress_cb,
-        )
     finally:
         s2 = _load_settings()
         for k, v in prev.items():
@@ -176,7 +140,7 @@ def _latest_inbox_fallback_uids(conn, reconnect):
         return [], reconnect()
 
 
-async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
     """Single pass of the auto-summarize/reply scan.
 
     When account_id is None, iterates over every enabled account in
@@ -203,41 +167,28 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
             names = {}
         if len(ids) <= 1:
             # Single-account (or zero rows — fallback to legacy settings.json lookup)
-            return await _auto_summarize_pass_single(
+            return await _auto_summarize_pass_single(days_back=days_back, account_id=(ids[0] if ids else None), progress_cb=progress_cb)
-                days_back=days_back,
-                account_id=(ids[0] if ids else None),
-                max_process=max_process,
-                progress_cb=progress_cb,
-            )
         outs = []
         for idx, aid in enumerate(ids, start=1):
             try:
                 await _emit_progress(progress_cb, f"{names.get(aid, aid[:8])}: starting ({idx}/{len(ids)})")
-                result = await _auto_summarize_pass_single(
+                result = await _auto_summarize_pass_single(days_back=days_back, account_id=aid, progress_cb=progress_cb)
-                    days_back=days_back,
-                    account_id=aid,
-                    max_process=max_process,
-                    progress_cb=progress_cb,
-                )
                 outs.append(f"[{names.get(aid, aid[:8])}] {result}")
             except Exception as e:
                 logger.warning(f"auto-summarize pass failed for account {aid}: {e}")
                 outs.append(f"[{names.get(aid, aid[:8])}] error: {e}")
         return "\n".join(outs)
-    return await _auto_summarize_pass_single(
+    return await _auto_summarize_pass_single(days_back=days_back, account_id=account_id, progress_cb=progress_cb)
-        days_back=days_back,
-        account_id=account_id,
-        max_process=max_process,
-        progress_cb=progress_cb,
-    )
 
 
-async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
     """Single pass of the auto-summarize/reply scan for ONE account.
     Reads current settings flags."""
     import asyncio
     import sqlite3 as _sql3
-    from src.llm_core import _uses_max_completion_tokens
+    import requests as _req
+    from src.endpoint_resolver import resolve_endpoint
+    from src.llm_core import _uses_max_completion_tokens, _restricts_temperature
 
     settings = _load_settings()
     auto_sum = settings.get("email_auto_summarize", False)
@@ -314,15 +265,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         ).fetchall()}
         if auto_tag or auto_spam:
             if account_owner:
-                _tag_existing = {r[0] for r in _c.execute(
+                _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner=?", (account_owner,)).fetchall()}
-                    "SELECT message_id FROM email_tags WHERE owner=? AND (account_id=? OR account_id='' OR account_id IS NULL)",
-                    (account_owner, account_id or ""),
-                ).fetchall()}
             else:
-                _tag_existing = {r[0] for r in _c.execute(
+                _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner='' OR owner IS NULL").fetchall()}
-                    "SELECT message_id FROM email_tags WHERE (owner='' OR owner IS NULL) AND (account_id=? OR account_id='' OR account_id IS NULL)",
-                    (account_id or "",),
-                ).fetchall()}
         else:
             _tag_existing = set()
         _cal_existing = {r[0] for r in _c.execute(
@@ -351,10 +296,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         if auto_spam and not spam_folder:
             logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move")
 
-        task_candidates = resolve_task_candidates(owner=account_owner)
+        url, model, headers = resolve_endpoint("utility", owner=account_owner)
-        if not task_candidates:
+        if not url:
+            url, model, headers = resolve_endpoint("default", owner=account_owner)
+        if not url or not model:
             return "No model configured"
-        url, model, headers = task_candidates[0]
 
         writing_style = settings.get("email_writing_style", "")
         processed = 0
@@ -368,14 +314,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         _reply_failed = 0
         _detail_lines = []
         _current_folder = "INBOX"
-        # Calendar extraction is sequential and each row can involve a model
+        _max_process = 5
-        # call plus a calendar write. Keep the scheduled calendar-only pass
-        # below the 5-minute action budget instead of timing out mid-run.
-        _default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5
-        try:
-            _max_process = max(1, int(max_process)) if max_process is not None else _default_max_process
-        except Exception:
-            _max_process = _default_max_process
         for _entry in uid_list:
             if processed >= _max_process:
                 break
@@ -467,30 +406,48 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     req_headers.update(headers)
 
                 if need_sum:
+                    tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
+                    payload = {
+                        "model": model,
+                        "messages": [
+                            {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
+                            {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
+                        ],
+                        tok_key: 16384,
+                        "temperature": 0.3,
+                        "stream": False,
+                    }
+                    # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
+                    if _restricts_temperature(model):
+                        payload.pop("temperature", None)
                     try:
-                        summary = await task_llm_call_async(
+                        # Use to_thread so this sync HTTP call doesn't freeze
-                            messages=[
+                        # the entire event loop while the LLM thinks (240s).
-                                {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
+                        resp = await asyncio.to_thread(
-                                {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
+                            _req.post, url, json=payload, headers=req_headers, timeout=240
-                            ],
-                            fallback_url=url, fallback_model=model, fallback_headers=headers,
-                            owner=account_owner or None,
-                            temperature=0.3, max_tokens=16384, timeout=240,
                         )
-                        summary = _extract_reply((summary or "").strip())
+                        if resp.ok:
-                        if summary:
+                            rdata = resp.json()
-                            _c = _sql3.connect(SCHEDULED_DB)
+                            m = (rdata.get("choices") or [{}])[0].get("message", {})
-                            _c.execute("""
+                            summary = (m.get("content") or "").strip()
-                                INSERT OR REPLACE INTO email_summaries
+                            summary = _extract_reply(summary)
-                                (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
+                            if not summary:
-                                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                                rc = (m.get("reasoning_content") or "").strip()
-                            """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
+                                bullets = [ln.strip() for ln in rc.split("\n") if re.match(r"^[-•*]\s+|^\d+[.)]\s+", ln.strip())]
-                            _c.commit()
+                                summary = "\n".join(bullets) if bullets else ""
-                            _c.close()
+                            if summary:
-                            _sum_existing.add(message_id)
+                                _c = _sql3.connect(SCHEDULED_DB)
-                            _summaries_created += 1
+                                _c.execute("""
-                            _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                                    INSERT OR REPLACE INTO email_summaries
-                            _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
+                                    (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
+                                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                                """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
+                                _c.commit()
+                                _c.close()
+                                _sum_existing.add(message_id)
+                                _summaries_created += 1
+                                _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                                _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
                     except Exception as e:
                         _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                         _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
@@ -511,14 +468,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     if context_snippets:
                         sys_prompt += "\n\nRELEVANT CONTEXT FROM PAST EMAILS AND CONTACTS:\n" + "\n\n---\n\n".join(context_snippets[:5])
                     try:
-                        reply = await task_llm_call_async(
+                        reply = await llm_call_async(
+                            url=url, model=model,
                             messages=[
                                 {"role": "system", "content": sys_prompt},
                                 {"role": "user", "content": f"Original email:\nFrom: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\nDraft a reply. Return only the reply body text."},
                             ],
-                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            temperature=0.7, max_tokens=1024,
-                            owner=account_owner or None,
+                            headers=req_headers, timeout=90,
-                            temperature=0.7, max_tokens=1024, timeout=90,
                         )
                         reply = _apply_email_style_mechanics(_extract_reply(reply or ""))
                         if reply:
@@ -545,8 +502,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                 # ── Calendar event extraction (independent of reply drafting) ──
                 if need_cal:
                     _cal_run_count = 0
-                    _cal_event_uids = []
-                    _cal_parse_ok = False
                     try:
                         # Pull a snapshot of upcoming events so the LLM can decide
                         # create vs update vs cancel based on what already exists.
@@ -555,7 +510,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                         _existing_summary = get_upcoming_events(_acct_owner, horizon_days=60, limit=40)
                         existing_json = json.dumps(_existing_summary)
                         is_sent = _folder.lower().startswith("sent") or "sent" in _folder.lower()
-                        cal_extract = await task_llm_call_async(
+                        cal_extract = await llm_call_async(
+                            url=url, model=model,
                             messages=[
                                 {"role": "system", "content": (
                                     "You are a calendar assistant. The user receives emails AND sends replies "
@@ -606,9 +562,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                     f"{body[:4000]}"
                                 )},
                             ],
-                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            temperature=0.1, max_tokens=16384,
-                            owner=account_owner or None,
+                            headers=req_headers, timeout=180,
-                            temperature=0.1, max_tokens=16384, timeout=75,
                         )
                         _raw_original = cal_extract or ""
                         cal_extract = _strip_think(_raw_original)
@@ -618,10 +573,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             if matches:
                                 cal_extract = matches[-1].group()
                         logger.info(f"[cal-extract] uid={uid.decode() if isinstance(uid, bytes) else uid} folder={_folder} subj={subject[:50]!r} raw_len={len(cal_extract)} orig_len={len(_raw_original)} raw={cal_extract[:800]!r}")
-                        ops = _extract_json_array_from_text(cal_extract)
+                        jm = re.search(r'\[.*\]', cal_extract, re.DOTALL)
-                        if ops is not None:
+                        if jm:
                             try:
-                                _cal_parse_ok = True
+                                ops = json.loads(jm.group())
                                 logger.info(f"[cal-extract] parsed {len(ops)} op(s)")
                                 if isinstance(ops, list) and ops:
                                     from src.tool_implementations import do_manage_calendar
@@ -651,8 +606,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                             r = await do_manage_calendar(json.dumps(args), owner=_acct_owner)
                                             if r.get("exit_code", 0) == 0:
                                                 logger.info(f"[cal-extract] Updated event uid={cuid} → {op.get('title')} {op['date']}")
-                                                if cuid and cuid not in _cal_event_uids:
-                                                    _cal_event_uids.append(cuid)
                                                 _cal_run_count += 1
                                             else:
                                                 logger.warning(f"[cal-extract] update failed: {r.get('error')}")
@@ -733,41 +686,29 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                                             r = await do_manage_calendar(cal_args, owner=_acct_owner)
                                             if r.get("exit_code", 0) == 0:
                                                 logger.info(f"[cal-extract] Created event: {op['title']} on {op['date']}")
-                                                _created_uid = (r.get("uid") or "").strip()
-                                                if _created_uid and _created_uid not in _cal_event_uids:
-                                                    _cal_event_uids.append(_created_uid)
                                                 _events_created += 1
                                                 _cal_run_count += 1
                                             else:
                                                 logger.warning(f"[cal-extract] create failed: {r.get('error')} args={cal_args[:200]}")
                             except Exception as je:
                                 logger.warning(f"[cal-extract] JSON parse failed: {je} on raw={cal_extract[:200]!r}")
-                        else:
-                            logger.warning(f"[cal-extract] no JSON array found on raw={cal_extract[:200]!r}")
                     except Exception as e:
                         logger.warning(f"[cal-extract] Meeting extraction LLM call failed for uid={uid}: {e}")
                     else:
-                        # Record successfully parsed results so we don't re-LLM
+                        # Record we processed this email so we don't re-LLM next run.
-                        # no-op emails. Transient LLM failures are retried on
+                        # Only mark as processed on success ? transient LLM failures
-                        # the next poll run.
+                        # are retried on the next poll run (matches summary/reply pattern).
                         try:
-                            if _cal_parse_ok:
+                            _cc = _sql3.connect(SCHEDULED_DB)
-                                _cc = _sql3.connect(SCHEDULED_DB)
+                            _cc.execute(
-                                _cc.execute(
+                                "INSERT OR REPLACE INTO email_calendar_extractions "
-                                    "INSERT OR REPLACE INTO email_calendar_extractions "
+                                "(message_id, owner, uid, events_created, created_at) VALUES (?, ?, ?, ?, ?)",
-                                    "(message_id, owner, uid, event_uids, events_created, created_at) VALUES (?, ?, ?, ?, ?, ?)",
+                                (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid),
-                                    (
+                                 _cal_run_count, datetime.utcnow().isoformat())
-                                        message_id,
+                            )
-                                        account_owner or "",
+                            _cc.commit()
-                                        uid.decode() if isinstance(uid, bytes) else str(uid),
+                            _cc.close()
-                                        json.dumps(_cal_event_uids),
+                            _cal_existing.add(message_id)
-                                        _cal_run_count,
-                                        datetime.utcnow().isoformat(),
-                                    ),
-                                )
-                                _cc.commit()
-                                _cc.close()
-                                _cal_existing.add(message_id)
                         except Exception as ce:
                             logger.debug(f"Could not cache calendar extraction: {ce}")
 
@@ -801,11 +742,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             "temperature": 0,
                             tok_key: 200,
                         }
-                        urg_raw = await task_llm_call_async(
+                        urg_raw = await llm_call_async(
-                            messages=payload["messages"],
+                            url=url, model=model, messages=payload["messages"],
-                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            temperature=0, max_tokens=200, headers=req_headers, timeout=60,
-                            owner=account_owner or None,
-                            temperature=0, max_tokens=200, timeout=60,
                         )
                         urg_raw = _strip_think(urg_raw or "")
                         urg_raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", urg_raw, flags=re.MULTILINE).strip()
@@ -906,13 +845,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                         class_sys = (
                             "Classify the email. Return ONLY a JSON object, no prose, no markdown fences. "
                             "Schema: {\"tags\": [\"tag1\"], \"spam\": false, \"reason\": \"short\"}. "
-                            "Pick 1-3 tags from: work, personal, urgent, action-needed, finance, bills, "
+                            "Pick 1-2 tags from: work, personal, finance, bills, receipt, travel, "
-                            "receipt, legal, travel, newsletter, promo, notification, security, social, "
+                            "newsletter, promo, notification, security, social, shopping, calendar.\n\n"
-                            "shopping, calendar, support.\n\n"
-                            "Use work for professional/company/client/operations messages. "
-                            "Use personal for friends/family/private-life messages. "
-                            "Use urgent for real time-sensitive consequences. "
-                            "Use action-needed when the user likely needs to reply, pay, sign, book, or decide.\n\n"
                             "Set spam=true for ANY of:\n"
                             "- Phishing, scams, chain mail, deceptive offers\n"
                             "- Marketing/promotional blasts (\"special offer\", \"limited time\", discount codes)\n"
@@ -929,55 +863,70 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             "If it's a mass-mailed generic update with no personal CTA, mark spam=true even if from a legitimate service. "
                             "Reason should be 5-10 words."
                         )
-                        raw_out = await task_llm_call_async(
+                        tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
-                            messages=[
+                        payload = {
+                            "model": model,
+                            "messages": [
                                 {"role": "system", "content": class_sys},
                                 {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body[:4000]}"},
                             ],
-                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                            tok_key: 512,
-                            owner=account_owner or None,
+                            "temperature": 0.1,
-                            temperature=0.1, max_tokens=512, timeout=120,
+                            "stream": False,
+                        }
+                        # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
+                        if _restricts_temperature(model):
+                            payload.pop("temperature", None)
+                        # to_thread keeps the event loop responsive during the LLM call
+                        resp = await asyncio.to_thread(
+                            _req.post, url, json=payload, headers=req_headers, timeout=120
                         )
-                        raw_out = _strip_think((raw_out or "").strip())
+                        if not resp.ok:
-                        raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
+                            logger.warning(f"Auto-classify {uid.decode() if isinstance(uid, bytes) else str(uid)} HTTP {resp.status_code}: {resp.text[:200]}")
-                        jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
+                        else:
-                        parsed = None
+                            rdata = resp.json()
-                        if jm:
+                            m = (rdata.get("choices") or [{}])[0].get("message", {})
-                            try:
+                            raw_out = (m.get("content") or "").strip()
-                                parsed = json.loads(jm.group(0))
+                            raw_out = _strip_think(raw_out)
-                            except Exception:
+                            raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
-                                parsed = None
+                            jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
-                        if parsed is not None:
+                            parsed = None
-                            _ALLOWED_TAGS = {"work","personal","urgent","action-needed","finance","bills",
+                            if jm:
-                                             "receipt","legal","travel","newsletter","marketing","notification",
+                                try:
-                                             "security","social","shopping","calendar","support"}
+                                    parsed = json.loads(jm.group(0))
-                            raw_tags = parsed.get("tags") or []
+                                except Exception:
-                            if isinstance(raw_tags, str):
+                                    parsed = None
-                                raw_tags = [raw_tags]
+                            if parsed is not None:
-                            tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
+                                _ALLOWED_TAGS = {"work","personal","finance","bills","receipt","travel",
-                            tags = ["marketing" if t == "promo" else t for t in tags]
+                                                 "newsletter","marketing","notification","security","social",
-                            tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
+                                                 "shopping","calendar"}
-                            is_spam = bool(parsed.get("spam"))
+                                raw_tags = parsed.get("tags") or []
-                            spam_reason = str(parsed.get("reason") or "")[:200]
+                                if isinstance(raw_tags, str):
+                                    raw_tags = [raw_tags]
+                                tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
+                                tags = ["marketing" if t == "promo" else t for t in tags]
+                                tags = [t for t in tags if t in _ALLOWED_TAGS][:2]
+                                is_spam = bool(parsed.get("spam"))
+                                spam_reason = str(parsed.get("reason") or "")[:200]
 
-                            moved_to = ""
+                                moved_to = ""
-                            if is_spam and auto_spam and spam_folder:
+                                if is_spam and auto_spam and spam_folder:
-                                if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
+                                    if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
-                                    moved_to = spam_folder
+                                        moved_to = spam_folder
-                                    logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
+                                        logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
 
-                            _c = _sql3.connect(SCHEDULED_DB)
+                                _c = _sql3.connect(SCHEDULED_DB)
-                            _c.execute("""
+                                _c.execute("""
-                                INSERT OR REPLACE INTO email_tags
+                                    INSERT OR REPLACE INTO email_tags
-                                (message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict,
+                                    (message_id, owner, uid, folder, subject, sender, tags, spam_verdict,
-                                 spam_reason, moved_to, model_used, created_at)
+                                     spam_reason, moved_to, model_used, created_at)
-                                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                                    VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?, ?, ?)
-                            """, (message_id, account_owner or "", account_id or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender,
+                                """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), subject, sender,
-                                  json.dumps(tags), 1 if is_spam else 0,
+                                      json.dumps(tags), 1 if is_spam else 0,
-                                  spam_reason, moved_to, model, datetime.utcnow().isoformat()))
+                                      spam_reason, moved_to, model, datetime.utcnow().isoformat()))
-                            _c.commit()
+                                _c.commit()
-                            _c.close()
+                                _c.close()
-                            _tag_existing.add(message_id)
+                                _tag_existing.add(message_id)
                     except Exception as e:
                         logger.warning(f"Auto-classify {uid} failed: {e}")
 

routes/gallery/gallery_helpers.py

@@ -99,7 +99,6 @@ def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any
         "filename": img.filename,
         "url": f"/api/generated-image/{img.filename}",
         "prompt": img.prompt,
-        "caption": img.caption or "",
         "model": img.model,
         "size": img.size,
         "quality": img.quality,

routes/gallery_helpers.py

@@ -1,9 +1,9 @@
-"""Backward-compat shim - canonical location is routes/gallery/gallery_helpers.py.
+"""Backward-compat shim — canonical location is routes/gallery/gallery_helpers.py.
 
 This module is replaced in ``sys.modules`` by the canonical module object so
 that ``import routes.gallery_helpers``, ``from routes.gallery_helpers import X``,
 ``importlib.import_module("routes.gallery_helpers")``, and
-``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the same
+``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the *same*
 object. Keeps existing import paths working after slice 2a (#4082/#4071).
 """
 

routes/gallery_routes.py

@@ -1,9 +1,9 @@
-"""Backward-compat shim - canonical location is routes/gallery/gallery_routes.py.
+"""Backward-compat shim — canonical location is routes/gallery/gallery_routes.py.
 
 This module is replaced in ``sys.modules`` by the canonical module object so
 that ``import routes.gallery_routes``, ``from routes.gallery_routes import X``,
 ``importlib.import_module("routes.gallery_routes")``, and
-``monkeypatch.setattr(routes.gallery_routes, ...)`` all operate on the same
+``monkeypatch.setattr(routes.gallery_routes, ...)`` all operate on the *same*
 object the application actually uses. Keeps existing import paths working
 after slice 2a (#4082/#4071). Source-introspection tests read the canonical
 file by path.

routes/history/__init__.py

@@ -1,5 +0,0 @@
-"""History route domain package (slice 2d, #4082/#4071).
-
-Contains history_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/history_routes.py re-exports from here.
-"""

routes/history/history_routes.py

@@ -1,768 +0,0 @@
-"""History routes — session history, truncation, fork, conversation topics."""
-
-import json
-import uuid
-import logging
-import re
-from typing import Dict, Any, Optional
-
-from fastapi import APIRouter, Request, HTTPException
-
-from core.models import ChatMessage
-from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
-from src.topic_analyzer import analyze_topics
-from routes.session_routes import (
-    _message_role,
-    _message_text,
-    _reject_compact_during_active_run,
-    _verify_session_owner,
-)
-
-logger = logging.getLogger(__name__)
-
-_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
-_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
-
-
-def _history_display_content(content: Any) -> Any:
-    """Return a lightweight browser-display copy of stored message content.
-
-    Older multimodal user messages may be persisted as a JSON *string*
-    containing image_url blocks with inline base64 image bytes. Those bytes are
-    needed for model calls when the turn is first sent, but they should not be
-    sent back through /api/history every time the user opens the chat. The
-    attachment metadata already carries file ids/names for the UI cards.
-    """
-    if isinstance(content, list):
-        text_parts = []
-        omitted_media = 0
-        for block in content:
-            if not isinstance(block, dict):
-                continue
-            if block.get("type") == "text":
-                text = block.get("text")
-                if isinstance(text, str) and text:
-                    text_parts.append(text)
-            elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
-                omitted_media += 1
-        text = "\n".join(text_parts).strip()
-        if omitted_media and not text:
-            return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
-        return text
-
-    if not isinstance(content, str):
-        return content
-    if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
-        return content
-
-    stripped = content.lstrip()
-    if stripped.startswith("["):
-        try:
-            blocks = json.loads(content)
-        except (json.JSONDecodeError, TypeError, ValueError):
-            blocks = None
-        if isinstance(blocks, list):
-            text_parts = []
-            for block in blocks:
-                if not isinstance(block, dict):
-                    continue
-                if block.get("type") == "text":
-                    text = block.get("text")
-                    if isinstance(text, str) and text:
-                        text_parts.append(text)
-            if text_parts:
-                return "\n".join(text_parts).strip()
-
-    if "data:image/" in content:
-        return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
-    return content
-
-
-def _merge_continue_rows_to_delete(db_messages, db1, db2):
-    """DB rows to delete when merging the last two assistant messages.
-
-    Always the second assistant message (db2), plus ONLY the single
-    intervening "continue" user message (the one carrying "previous response
-    was interrupted") — matching the in-memory merge. The previous code
-    deleted the whole index range between the two assistant rows, destroying
-    any tool/system/user messages in between and desyncing the DB from the
-    in-memory history.
-    """
-    to_delete = [db2]
-    i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
-    i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
-    if i1 is not None and i2 is not None and i2 - 1 > i1:
-        between = db_messages[i2 - 1]
-        if getattr(between, "role", "") == "user" and            "previous response was interrupted" in (getattr(between, "content", "") or ""):
-            to_delete.append(between)
-    return to_delete
-
-
-def setup_history_routes(session_manager) -> APIRouter:
-    router = APIRouter(tags=["history"])
-
-    def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
-        entry = {"role": m.role, "content": _history_display_content(m.content)}
-        meta = {}
-        if m.meta_data:
-            try:
-                meta = json.loads(m.meta_data) or {}
-            except (json.JSONDecodeError, ValueError):
-                meta = {}
-        if m.timestamp and "timestamp" not in meta:
-            meta["timestamp"] = m.timestamp.isoformat() + "Z"
-        if meta:
-            entry["metadata"] = meta
-        return entry
-
-    @router.get("/api/history/{session_id}")
-    async def get_session_history(
-        request: Request,
-        session_id: str,
-        limit: Optional[int] = None,
-        offset: Optional[int] = None,
-    ) -> Dict[str, Any]:
-        _verify_session_owner(request, session_id)
-        if limit is not None:
-            page_limit = max(1, min(int(limit), 100))
-            db = SessionLocal()
-            try:
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session is None:
-                    raise HTTPException(404, f"Session '{session_id}' not found")
-
-                total = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .count()
-                )
-                page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
-                page_offset = max(0, min(page_offset, total))
-                rows = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .offset(page_offset)
-                    .limit(page_limit)
-                    .all()
-                )
-                history_dict = [
-                    entry for entry in (_db_history_entry(m) for m in rows)
-                    if not (entry.get("metadata") or {}).get("hidden")
-                ]
-                return {
-                    "history": history_dict,
-                    "model": db_session.model,
-                    "endpoint_url": db_session.endpoint_url,
-                    "name": db_session.name,
-                    "offset": page_offset,
-                    "limit": page_limit,
-                    "total": total,
-                    "has_more_before": page_offset > 0,
-                    "has_more_after": page_offset + len(rows) < total,
-                }
-            finally:
-                db.close()
-
-        try:
-            session = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session '{session_id}' not found")
-
-        history_dict = []
-        for msg in session.history:
-            if isinstance(msg, ChatMessage):
-                # Skip hidden messages (e.g. compaction summaries for AI context)
-                if msg.metadata and msg.metadata.get("hidden"):
-                    continue
-                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
-                if msg.metadata:
-                    entry["metadata"] = msg.metadata
-                history_dict.append(entry)
-            elif isinstance(msg, dict):
-                if msg.get("metadata", {}).get("hidden"):
-                    continue
-                entry = {
-                    "role": msg.get("role", ""),
-                    "content": _history_display_content(msg.get("content", "")),
-                }
-                if msg.get("metadata"):
-                    entry["metadata"] = msg["metadata"]
-                history_dict.append(entry)
-
-        # Fallback: load from DB if in-memory is empty
-        if not history_dict:
-            db = SessionLocal()
-            try:
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .all()
-                )
-                db_history = []
-                for m in db_messages:
-                    db_history.append(_db_history_entry(m))
-                if db_history:
-                    # Rebuild in-memory history from the full set so hidden
-                    # messages (e.g. compaction summaries) are kept for AI context.
-                    session.history = [
-                        ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
-                        for m in db_history
-                    ]
-                # Response excludes hidden messages, matching the in-memory path.
-                history_dict = [
-                    m for m in db_history
-                    if not (m.get("metadata") or {}).get("hidden")
-                ]
-            except Exception as e:
-                logger.error(f"DB fallback failed for {session_id}: {e}")
-            finally:
-                db.close()
-
-        return {
-            "history": history_dict,
-            "model": session.model,
-            "endpoint_url": session.endpoint_url,
-            "name": session.name,
-        }
-
-    @router.post("/api/session/{session_id}/truncate")
-    async def truncate_session(request: Request, session_id: str):
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            keep_count = body.get("keep_count", 0)
-            result = session_manager.truncate_messages(session_id, keep_count)
-            return {"status": "ok", "kept": keep_count, "truncated": result}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Truncate error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/message")
-    async def add_message(request: Request, session_id: str):
-        """Add a message to a session (for slash command persistence)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            role = body.get("role", "assistant")
-            content = body.get("content", "")
-            if not content:
-                raise HTTPException(400, "content is required")
-            msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
-            session_manager.add_message(session_id, msg)
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-
-    @router.post("/api/session/{session_id}/delete-messages")
-    async def delete_messages(request: Request, session_id: str):
-        """Delete specific messages by DB ID (or legacy index)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            msg_ids = body.get("msg_ids", [])
-            indices = body.get("indices")  # legacy fallback
-
-            session = session_manager.get_session(session_id)
-            db = SessionLocal()
-            try:
-                if msg_ids:
-                    # New ID-based delete
-                    deleted = 0
-                    for mid in msg_ids:
-                        db_msg = db.query(DbChatMessage).filter(
-                            DbChatMessage.id == mid,
-                            DbChatMessage.session_id == session_id,
-                        ).first()
-                        if db_msg:
-                            db.delete(db_msg)
-                            deleted += 1
-
-                    # Remove from in-memory history by matching _db_id
-                    def _get_db_id(m):
-                        meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
-                        return meta.get('_db_id') if isinstance(meta, dict) else None
-                    session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
-                elif indices:
-                    # Legacy index-based delete
-                    indices = sorted(indices, reverse=True)
-                    db_messages = db.query(DbChatMessage).filter(
-                        DbChatMessage.session_id == session_id
-                    ).order_by(DbChatMessage.timestamp).all()
-
-                    deleted = 0
-                    for idx in indices:
-                        if 0 <= idx < len(db_messages):
-                            db.delete(db_messages[idx])
-                            deleted += 1
-                        if 0 <= idx < len(session.history):
-                            session.history.pop(idx)
-                else:
-                    return {"status": "ok", "deleted": 0}
-
-                session.message_count = len(session.history)
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session:
-                    db_session.message_count = len(session.history)
-                    from datetime import datetime, timezone
-                    db_session.updated_at = datetime.now(timezone.utc)
-
-                db.commit()
-                return {"status": "ok", "deleted": deleted}
-            finally:
-                db.close()
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Delete messages error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/edit-message")
-    async def edit_message(request: Request, session_id: str):
-        """Edit the content of a message by its database ID."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            msg_id = body.get("msg_id")
-            content = body.get("content")
-            if not msg_id or content is None:
-                raise HTTPException(400, "msg_id and content are required")
-
-            session = session_manager.get_session(session_id)
-            db = SessionLocal()
-            try:
-                db_msg = db.query(DbChatMessage).filter(
-                    DbChatMessage.id == msg_id,
-                    DbChatMessage.session_id == session_id,
-                ).first()
-                if not db_msg:
-                    raise HTTPException(404, "Message not found")
-
-                db_msg.content = content
-                meta = {}
-                if db_msg.meta_data:
-                    try: meta = json.loads(db_msg.meta_data)
-                    except (json.JSONDecodeError, ValueError): pass
-                meta['edited'] = True
-                db_msg.meta_data = json.dumps(meta)
-
-                # Update in-memory history by matching _db_id
-                for hmsg in session.history:
-                    hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
-                    if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
-                        if isinstance(hmsg, ChatMessage):
-                            hmsg.content = content
-                            hmsg.metadata['edited'] = True
-                        elif isinstance(hmsg, dict):
-                            hmsg['content'] = content
-                            hmsg['metadata']['edited'] = True
-                        break
-
-                db.commit()
-                return {"status": "ok"}
-            finally:
-                db.close()
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except HTTPException:
-            raise
-        except Exception as e:
-            logger.error(f"Edit message error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/mark-stopped")
-    async def mark_stopped(request: Request, session_id: str):
-        """Mark the last assistant message as stopped by user."""
-        _verify_session_owner(request, session_id)
-        try:
-            session = session_manager.get_session(session_id)
-            # Find last assistant message and add stopped metadata
-            for msg in reversed(session.history):
-                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
-                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
-                    if isinstance(msg, ChatMessage):
-                        if not msg.metadata:
-                            msg.metadata = {}
-                        msg.metadata['stopped'] = True
-                        if not msg.metadata.get('model'):
-                            msg.metadata['model'] = session.model
-                    else:
-                        if 'metadata' not in msg:
-                            msg['metadata'] = {}
-                        msg['metadata']['stopped'] = True
-                        if not msg['metadata'].get('model'):
-                            msg['metadata']['model'] = session.model
-                    break
-            # Also update in DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
-                    .order_by(DbChatMessage.timestamp.desc())
-                    .first()
-                )
-                if db_messages:
-                    meta = {}
-                    if db_messages.meta_data:
-                        try:
-                            meta = _json.loads(db_messages.meta_data)
-                        except (json.JSONDecodeError, ValueError):
-                            pass
-                    meta['stopped'] = True
-                    if not meta.get('model'):
-                        meta['model'] = session.model
-                    db_messages.meta_data = _json.dumps(meta)
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Mark stopped error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/update-last-meta")
-    async def update_last_meta(request: Request, session_id: str):
-        """Merge metadata into the last assistant message (e.g. save variants)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            meta_update = body.get("metadata", {})
-            session = session_manager.get_session(session_id)
-
-            # Update in-memory
-            for msg in reversed(session.history):
-                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
-                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
-                    if isinstance(msg, ChatMessage):
-                        if not msg.metadata:
-                            msg.metadata = {}
-                        msg.metadata.update(meta_update)
-                    else:
-                        if 'metadata' not in msg:
-                            msg['metadata'] = {}
-                        msg['metadata'].update(meta_update)
-                    break
-
-            # Update in DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_msg = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
-                    .order_by(DbChatMessage.timestamp.desc())
-                    .first()
-                )
-                if db_msg:
-                    meta = {}
-                    if db_msg.meta_data:
-                        try: meta = _json.loads(db_msg.meta_data)
-                        except (json.JSONDecodeError, ValueError): pass
-                    meta.update(meta_update)
-                    db_msg.meta_data = _json.dumps(meta)
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Update last meta error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/merge-last-assistant")
-    async def merge_last_assistant(request: Request, session_id: str):
-        """Merge the last two assistant messages into one (for continue)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            separator = body.get("separator", "\n\n")
-            session = session_manager.get_session(session_id)
-
-            # Find last two assistant messages in-memory
-            ai_indices = []
-            for i, msg in enumerate(session.history):
-                role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
-                if role == 'assistant':
-                    ai_indices.append(i)
-
-            if len(ai_indices) < 2:
-                return {"status": "ok", "merged": False}
-
-            idx1, idx2 = ai_indices[-2], ai_indices[-1]
-            msg1, msg2 = session.history[idx1], session.history[idx2]
-
-            content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
-            content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
-            merged_content = content1 + separator + content2
-
-            # Merge metadata
-            meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
-            meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
-            merged_meta = {**meta1, **meta2}
-            merged_meta.pop('stopped', None)  # no longer stopped after continue
-
-            # Update first message, remove second
-            if isinstance(msg1, ChatMessage):
-                msg1.content = merged_content
-                msg1.metadata = merged_meta
-            else:
-                msg1['content'] = merged_content
-                msg1['metadata'] = merged_meta
-
-            # Also remove the hidden "continue" user message between them if present
-            # It's the message at idx2-1 if it's a user message with continue text
-            remove_indices = [idx2]
-            if idx2 - 1 > idx1:
-                between = session.history[idx2 - 1]
-                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
-                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
-                if between_role == 'user' and 'previous response was interrupted' in between_content:
-                    remove_indices.insert(0, idx2 - 1)
-
-            for ri in sorted(remove_indices, reverse=True):
-                session.history.pop(ri)
-
-            # Update DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .all()
-                )
-                # Find last two assistant messages in DB
-                ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
-                if len(ai_db) >= 2:
-                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
-                    db1.content = merged_content
-                    db1.meta_data = _json.dumps(merged_meta)
-
-                    # Mirror the in-memory deletion: remove the second assistant
-                    # message and ONLY the "continue" user message between them
-                    # (not arbitrary tool/system/user rows). The old
-                    # range-delete destroyed every row between the two assistant
-                    # messages, desyncing the DB from the in-memory history.
-                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
-                        db.delete(_row)
-
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok", "merged": True}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Merge assistant error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/fork")
-    async def fork_session(request: Request, session_id: str):
-        """Create a new session with messages copied up to keep_count."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            keep_count = body.get("keep_count", 0)
-
-            # Get the source session
-            source = session_manager.sessions.get(session_id)
-            if not source:
-                raise HTTPException(404, "Session not found")
-
-            # Create new session
-            new_id = str(uuid.uuid4())
-            fork_name = f"\u2ADD {source.name}"
-            new_session = session_manager.create_session(
-                session_id=new_id,
-                name=fork_name,
-                endpoint_url=source.endpoint_url,
-                model=source.model,
-                rag=False,
-                owner=getattr(source, 'owner', None),
-            )
-
-            # Copy messages up to keep_count
-            msgs_to_copy = source.history[:keep_count]
-            for msg in msgs_to_copy:
-                # Copy the metadata dict. Sharing it would let the fork's
-                # persistence (add_message -> _persist_message stamps
-                # _db_id/timestamp onto the dict) mutate the SOURCE session's
-                # in-memory messages, corrupting their _db_id and breaking
-                # edit/delete-by-id on the original conversation.
-                meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
-                new_session.add_message(ChatMessage(msg.role, msg.content, meta))
-            try:
-                from src.event_bus import fire_event
-                fire_event("session_created", getattr(source, 'owner', None))
-            except Exception:
-                logger.debug("session_created event dispatch failed", exc_info=True)
-
-            return {
-                "status": "ok",
-                "id": new_id,
-                "name": fork_name,
-                "kept": len(msgs_to_copy),
-            }
-        except HTTPException:
-            raise
-        except Exception as e:
-            logger.error(f"Fork error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.get("/api/conversations/topics")
-    async def get_conversation_topics(request: Request) -> Dict[str, Any]:
-        from src.auth_helpers import require_user
-        user = require_user(request)
-        try:
-            return analyze_topics(session_manager, owner=user or None)
-        except Exception as e:
-            raise HTTPException(500, f"Topic analysis failed: {e}")
-
-    @router.post("/api/session/{session_id}/compact")
-    async def compact_session(request: Request, session_id: str):
-        """Manually trigger context compaction for a session."""
-        _verify_session_owner(request, session_id)
-        from src.auth_helpers import effective_user
-        owner = effective_user(request)
-        try:
-            session = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        _reject_compact_during_active_run(session_id)
-
-        try:
-            from src.model_context import estimate_tokens, get_context_length
-            from src.llm_core import llm_call_async
-            from src.endpoint_resolver import resolve_endpoint
-
-            if len(session.history) < 6:
-                return {"status": "ok", "message": "Not enough messages to compact"}
-
-            ctx_len = get_context_length(session.endpoint_url, session.model)
-            messages_before = session.get_context_messages()
-            used_before = estimate_tokens(messages_before)
-            pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
-            msg_count_before = len(session.history)
-
-            # Keep only last 4 messages, summarize the rest
-            keep_count = 4
-            older = session.history[:-keep_count]
-            recent = session.history[-keep_count:]
-
-            # Build text to summarize
-            convo_text = "\n".join(
-                f"{_message_role(m).upper()}: "
-                f"{_message_text(m)[:2000]}"
-                for m in older
-            )
-
-            # Use utility model if available
-            util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
-            compact_url = util_url or session.endpoint_url
-            compact_model = util_model or session.model
-            compact_headers = util_headers if util_url else session.headers
-
-            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
-            compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
-            sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
-            summary = await llm_call_async(
-                compact_url, compact_model,
-                [
-                    {"role": "system", "content": sys_prompt},
-                    {"role": "user", "content": convo_text},
-                ],
-                temperature=0.2, max_tokens=1024,
-                headers=compact_headers, timeout=30,
-            )
-
-            # Replace session history: summary as system message + recent messages
-            # System message holds the full summary for AI context
-            system_summary = ChatMessage(
-                role="system",
-                content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
-                metadata={"compacted": True, "hidden": True},
-            )
-            # Visible assistant message just shows stats
-            summary_msg = ChatMessage(
-                role="assistant",
-                content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
-                metadata={"compacted": True, "messages_removed": len(older)},
-            )
-            new_history = [system_summary, summary_msg] + list(recent)
-            session.history = new_history
-            session.message_count = len(session.history)
-            logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
-
-            # Update DB: delete old messages, insert summary
-            db = SessionLocal()
-            try:
-                db_msgs = db.query(DbChatMessage).filter(
-                    DbChatMessage.session_id == session_id
-                ).order_by(DbChatMessage.timestamp).all()
-
-                # Delete all but the last keep_count
-                for m in db_msgs[:-keep_count]:
-                    db.delete(m)
-
-                # Insert system summary (hidden, for AI context) and visible summary
-                import json as _json
-                import uuid
-                from datetime import datetime, timezone
-                now = datetime.now(timezone.utc)
-                db_sys_summary = DbChatMessage(
-                    id=str(uuid.uuid4()),
-                    session_id=session_id,
-                    role="system",
-                    content=system_summary.content,
-                    meta_data=_json.dumps(system_summary.metadata),
-                    timestamp=now,
-                )
-                db.add(db_sys_summary)
-                db_summary = DbChatMessage(
-                    id=str(uuid.uuid4()),
-                    session_id=session_id,
-                    role="assistant",
-                    content=summary_msg.content,
-                    meta_data=_json.dumps(summary_msg.metadata),
-                    timestamp=now,
-                )
-                db.add(db_summary)
-
-                # Update session record
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session:
-                    db_session.message_count = len(session.history)
-                    db_session.updated_at = datetime.now(timezone.utc)
-                db.commit()
-            finally:
-                db.close()
-
-            session_manager.save_sessions()
-
-            used_after = estimate_tokens(session.get_context_messages())
-            pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
-
-            return {
-                "status": "ok",
-                "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
-                "before": pct_before,
-                "after": pct_after,
-            }
-
-        except Exception as e:
-            logger.error(f"Manual compact error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    return router

routes/history_routes.py

@@ -1,17 +1,662 @@
-"""Backward-compat shim — canonical location is routes/history/history_routes.py.
+"""History routes — session history, truncation, fork, conversation topics."""
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+import json
-that ``import routes.history_routes``, ``from routes.history_routes import X``,
+import uuid
-``importlib.import_module("routes.history_routes")``, and the
+import logging
-``import ... as history_routes`` + ``monkeypatch.setattr(history_routes, ...)``
+from typing import Dict, Any
-pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py
-all operate on the *same* object the application actually uses. Keeps existing
-import paths working after slice 2d (#4082/#4071). Source-introspection tests
-read the canonical file by path.
-"""
 
-import sys as _sys
+from fastapi import APIRouter, Request, HTTPException
 
-from routes.history import history_routes as _canonical  # noqa: F401
+from core.models import ChatMessage
+from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
+from src.topic_analyzer import analyze_topics
+from routes.session_routes import (
+    _message_role,
+    _message_text,
+    _reject_compact_during_active_run,
+    _verify_session_owner,
+)
 
-_sys.modules[__name__] = _canonical
+logger = logging.getLogger(__name__)
+
+
+def _merge_continue_rows_to_delete(db_messages, db1, db2):
+    """DB rows to delete when merging the last two assistant messages.
+
+    Always the second assistant message (db2), plus ONLY the single
+    intervening "continue" user message (the one carrying "previous response
+    was interrupted") — matching the in-memory merge. The previous code
+    deleted the whole index range between the two assistant rows, destroying
+    any tool/system/user messages in between and desyncing the DB from the
+    in-memory history.
+    """
+    to_delete = [db2]
+    i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
+    i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
+    if i1 is not None and i2 is not None and i2 - 1 > i1:
+        between = db_messages[i2 - 1]
+        if getattr(between, "role", "") == "user" and            "previous response was interrupted" in (getattr(between, "content", "") or ""):
+            to_delete.append(between)
+    return to_delete
+
+
+def setup_history_routes(session_manager) -> APIRouter:
+    router = APIRouter(tags=["history"])
+
+    @router.get("/api/history/{session_id}")
+    async def get_session_history(request: Request, session_id: str) -> Dict[str, Any]:
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, f"Session '{session_id}' not found")
+
+        history_dict = []
+        for msg in session.history:
+            if isinstance(msg, ChatMessage):
+                # Skip hidden messages (e.g. compaction summaries for AI context)
+                if msg.metadata and msg.metadata.get("hidden"):
+                    continue
+                entry = {"role": msg.role, "content": msg.content}
+                if msg.metadata:
+                    entry["metadata"] = msg.metadata
+                history_dict.append(entry)
+            elif isinstance(msg, dict):
+                if msg.get("metadata", {}).get("hidden"):
+                    continue
+                entry = {
+                    "role": msg.get("role", ""),
+                    "content": msg.get("content", ""),
+                }
+                if msg.get("metadata"):
+                    entry["metadata"] = msg["metadata"]
+                history_dict.append(entry)
+
+        # Fallback: load from DB if in-memory is empty
+        if not history_dict:
+            db = SessionLocal()
+            try:
+                db_messages = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .all()
+                )
+                import json as _json
+                db_history = []
+                for m in db_messages:
+                    entry = {"role": m.role, "content": m.content}
+                    meta = {}
+                    if m.meta_data:
+                        try:
+                            meta = _json.loads(m.meta_data) or {}
+                        except (json.JSONDecodeError, ValueError):
+                            meta = {}
+                    if m.timestamp and "timestamp" not in meta:
+                        meta["timestamp"] = m.timestamp.isoformat() + "Z"
+                    if meta:
+                        entry["metadata"] = meta
+                    db_history.append(entry)
+                if db_history:
+                    # Rebuild in-memory history from the full set so hidden
+                    # messages (e.g. compaction summaries) are kept for AI context.
+                    session.history = [
+                        ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
+                        for m in db_history
+                    ]
+                # Response excludes hidden messages, matching the in-memory path.
+                history_dict = [
+                    m for m in db_history
+                    if not (m.get("metadata") or {}).get("hidden")
+                ]
+            except Exception as e:
+                logger.error(f"DB fallback failed for {session_id}: {e}")
+            finally:
+                db.close()
+
+        return {
+            "history": history_dict,
+            "model": session.model,
+            "endpoint_url": session.endpoint_url,
+            "name": session.name,
+        }
+
+    @router.post("/api/session/{session_id}/truncate")
+    async def truncate_session(request: Request, session_id: str):
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            keep_count = body.get("keep_count", 0)
+            result = session_manager.truncate_messages(session_id, keep_count)
+            return {"status": "ok", "kept": keep_count, "truncated": result}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Truncate error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/message")
+    async def add_message(request: Request, session_id: str):
+        """Add a message to a session (for slash command persistence)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            role = body.get("role", "assistant")
+            content = body.get("content", "")
+            if not content:
+                raise HTTPException(400, "content is required")
+            msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
+            session_manager.add_message(session_id, msg)
+            return {"status": "ok"}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+    @router.post("/api/session/{session_id}/delete-messages")
+    async def delete_messages(request: Request, session_id: str):
+        """Delete specific messages by DB ID (or legacy index)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            msg_ids = body.get("msg_ids", [])
+            indices = body.get("indices")  # legacy fallback
+
+            session = session_manager.get_session(session_id)
+            db = SessionLocal()
+            try:
+                if msg_ids:
+                    # New ID-based delete
+                    deleted = 0
+                    for mid in msg_ids:
+                        db_msg = db.query(DbChatMessage).filter(
+                            DbChatMessage.id == mid,
+                            DbChatMessage.session_id == session_id,
+                        ).first()
+                        if db_msg:
+                            db.delete(db_msg)
+                            deleted += 1
+
+                    # Remove from in-memory history by matching _db_id
+                    def _get_db_id(m):
+                        meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
+                        return meta.get('_db_id') if isinstance(meta, dict) else None
+                    session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
+                elif indices:
+                    # Legacy index-based delete
+                    indices = sorted(indices, reverse=True)
+                    db_messages = db.query(DbChatMessage).filter(
+                        DbChatMessage.session_id == session_id
+                    ).order_by(DbChatMessage.timestamp).all()
+
+                    deleted = 0
+                    for idx in indices:
+                        if 0 <= idx < len(db_messages):
+                            db.delete(db_messages[idx])
+                            deleted += 1
+                        if 0 <= idx < len(session.history):
+                            session.history.pop(idx)
+                else:
+                    return {"status": "ok", "deleted": 0}
+
+                session.message_count = len(session.history)
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session:
+                    db_session.message_count = len(session.history)
+                    from datetime import datetime, timezone
+                    db_session.updated_at = datetime.now(timezone.utc)
+
+                db.commit()
+                return {"status": "ok", "deleted": deleted}
+            finally:
+                db.close()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Delete messages error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/edit-message")
+    async def edit_message(request: Request, session_id: str):
+        """Edit the content of a message by its database ID."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            msg_id = body.get("msg_id")
+            content = body.get("content")
+            if not msg_id or content is None:
+                raise HTTPException(400, "msg_id and content are required")
+
+            session = session_manager.get_session(session_id)
+            db = SessionLocal()
+            try:
+                db_msg = db.query(DbChatMessage).filter(
+                    DbChatMessage.id == msg_id,
+                    DbChatMessage.session_id == session_id,
+                ).first()
+                if not db_msg:
+                    raise HTTPException(404, "Message not found")
+
+                db_msg.content = content
+                meta = {}
+                if db_msg.meta_data:
+                    try: meta = json.loads(db_msg.meta_data)
+                    except (json.JSONDecodeError, ValueError): pass
+                meta['edited'] = True
+                db_msg.meta_data = json.dumps(meta)
+
+                # Update in-memory history by matching _db_id
+                for hmsg in session.history:
+                    hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
+                    if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
+                        if isinstance(hmsg, ChatMessage):
+                            hmsg.content = content
+                            hmsg.metadata['edited'] = True
+                        elif isinstance(hmsg, dict):
+                            hmsg['content'] = content
+                            hmsg['metadata']['edited'] = True
+                        break
+
+                db.commit()
+                return {"status": "ok"}
+            finally:
+                db.close()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Edit message error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/mark-stopped")
+    async def mark_stopped(request: Request, session_id: str):
+        """Mark the last assistant message as stopped by user."""
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+            # Find last assistant message and add stopped metadata
+            for msg in reversed(session.history):
+                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
+                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
+                    if isinstance(msg, ChatMessage):
+                        if not msg.metadata:
+                            msg.metadata = {}
+                        msg.metadata['stopped'] = True
+                        if not msg.metadata.get('model'):
+                            msg.metadata['model'] = session.model
+                    else:
+                        if 'metadata' not in msg:
+                            msg['metadata'] = {}
+                        msg['metadata']['stopped'] = True
+                        if not msg['metadata'].get('model'):
+                            msg['metadata']['model'] = session.model
+                    break
+            # Also update in DB
+            db = SessionLocal()
+            try:
+                import json as _json
+                db_messages = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
+                    .order_by(DbChatMessage.timestamp.desc())
+                    .first()
+                )
+                if db_messages:
+                    meta = {}
+                    if db_messages.meta_data:
+                        try:
+                            meta = _json.loads(db_messages.meta_data)
+                        except (json.JSONDecodeError, ValueError):
+                            pass
+                    meta['stopped'] = True
+                    if not meta.get('model'):
+                        meta['model'] = session.model
+                    db_messages.meta_data = _json.dumps(meta)
+                    db.commit()
+            finally:
+                db.close()
+            session_manager.save_sessions()
+            return {"status": "ok"}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Mark stopped error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/update-last-meta")
+    async def update_last_meta(request: Request, session_id: str):
+        """Merge metadata into the last assistant message (e.g. save variants)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            meta_update = body.get("metadata", {})
+            session = session_manager.get_session(session_id)
+
+            # Update in-memory
+            for msg in reversed(session.history):
+                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
+                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
+                    if isinstance(msg, ChatMessage):
+                        if not msg.metadata:
+                            msg.metadata = {}
+                        msg.metadata.update(meta_update)
+                    else:
+                        if 'metadata' not in msg:
+                            msg['metadata'] = {}
+                        msg['metadata'].update(meta_update)
+                    break
+
+            # Update in DB
+            db = SessionLocal()
+            try:
+                import json as _json
+                db_msg = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
+                    .order_by(DbChatMessage.timestamp.desc())
+                    .first()
+                )
+                if db_msg:
+                    meta = {}
+                    if db_msg.meta_data:
+                        try: meta = _json.loads(db_msg.meta_data)
+                        except (json.JSONDecodeError, ValueError): pass
+                    meta.update(meta_update)
+                    db_msg.meta_data = _json.dumps(meta)
+                    db.commit()
+            finally:
+                db.close()
+            session_manager.save_sessions()
+            return {"status": "ok"}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Update last meta error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/merge-last-assistant")
+    async def merge_last_assistant(request: Request, session_id: str):
+        """Merge the last two assistant messages into one (for continue)."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            separator = body.get("separator", "\n\n")
+            session = session_manager.get_session(session_id)
+
+            # Find last two assistant messages in-memory
+            ai_indices = []
+            for i, msg in enumerate(session.history):
+                role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
+                if role == 'assistant':
+                    ai_indices.append(i)
+
+            if len(ai_indices) < 2:
+                return {"status": "ok", "merged": False}
+
+            idx1, idx2 = ai_indices[-2], ai_indices[-1]
+            msg1, msg2 = session.history[idx1], session.history[idx2]
+
+            content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
+            content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
+            merged_content = content1 + separator + content2
+
+            # Merge metadata
+            meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
+            meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
+            merged_meta = {**meta1, **meta2}
+            merged_meta.pop('stopped', None)  # no longer stopped after continue
+
+            # Update first message, remove second
+            if isinstance(msg1, ChatMessage):
+                msg1.content = merged_content
+                msg1.metadata = merged_meta
+            else:
+                msg1['content'] = merged_content
+                msg1['metadata'] = merged_meta
+
+            # Also remove the hidden "continue" user message between them if present
+            # It's the message at idx2-1 if it's a user message with continue text
+            remove_indices = [idx2]
+            if idx2 - 1 > idx1:
+                between = session.history[idx2 - 1]
+                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
+                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
+                if between_role == 'user' and 'previous response was interrupted' in between_content:
+                    remove_indices.insert(0, idx2 - 1)
+
+            for ri in sorted(remove_indices, reverse=True):
+                session.history.pop(ri)
+
+            # Update DB
+            db = SessionLocal()
+            try:
+                import json as _json
+                db_messages = (
+                    db.query(DbChatMessage)
+                    .filter(DbChatMessage.session_id == session_id)
+                    .order_by(DbChatMessage.timestamp)
+                    .all()
+                )
+                # Find last two assistant messages in DB
+                ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
+                if len(ai_db) >= 2:
+                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
+                    db1.content = merged_content
+                    db1.meta_data = _json.dumps(merged_meta)
+
+                    # Mirror the in-memory deletion: remove the second assistant
+                    # message and ONLY the "continue" user message between them
+                    # (not arbitrary tool/system/user rows). The old
+                    # range-delete destroyed every row between the two assistant
+                    # messages, desyncing the DB from the in-memory history.
+                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
+                        db.delete(_row)
+
+                    db.commit()
+            finally:
+                db.close()
+            session_manager.save_sessions()
+            return {"status": "ok", "merged": True}
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        except Exception as e:
+            logger.error(f"Merge assistant error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.post("/api/session/{session_id}/fork")
+    async def fork_session(request: Request, session_id: str):
+        """Create a new session with messages copied up to keep_count."""
+        _verify_session_owner(request, session_id)
+        try:
+            body = await request.json()
+            keep_count = body.get("keep_count", 0)
+
+            # Get the source session
+            source = session_manager.sessions.get(session_id)
+            if not source:
+                raise HTTPException(404, "Session not found")
+
+            # Create new session
+            new_id = str(uuid.uuid4())
+            fork_name = f"\u2ADD {source.name}"
+            new_session = session_manager.create_session(
+                session_id=new_id,
+                name=fork_name,
+                endpoint_url=source.endpoint_url,
+                model=source.model,
+                rag=False,
+                owner=getattr(source, 'owner', None),
+            )
+
+            # Copy messages up to keep_count
+            msgs_to_copy = source.history[:keep_count]
+            for msg in msgs_to_copy:
+                # Copy the metadata dict. Sharing it would let the fork's
+                # persistence (add_message -> _persist_message stamps
+                # _db_id/timestamp onto the dict) mutate the SOURCE session's
+                # in-memory messages, corrupting their _db_id and breaking
+                # edit/delete-by-id on the original conversation.
+                meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
+                new_session.add_message(ChatMessage(msg.role, msg.content, meta))
+            try:
+                from src.event_bus import fire_event
+                fire_event("session_created", getattr(source, 'owner', None))
+            except Exception:
+                logger.debug("session_created event dispatch failed", exc_info=True)
+
+            return {
+                "status": "ok",
+                "id": new_id,
+                "name": fork_name,
+                "kept": len(msgs_to_copy),
+            }
+        except HTTPException:
+            raise
+        except Exception as e:
+            logger.error(f"Fork error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    @router.get("/api/conversations/topics")
+    async def get_conversation_topics(request: Request) -> Dict[str, Any]:
+        from src.auth_helpers import require_user
+        user = require_user(request)
+        try:
+            return analyze_topics(session_manager, owner=user or None)
+        except Exception as e:
+            raise HTTPException(500, f"Topic analysis failed: {e}")
+
+    @router.post("/api/session/{session_id}/compact")
+    async def compact_session(request: Request, session_id: str):
+        """Manually trigger context compaction for a session."""
+        _verify_session_owner(request, session_id)
+        from src.auth_helpers import effective_user
+        owner = effective_user(request)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        _reject_compact_during_active_run(session_id)
+
+        try:
+            from src.model_context import estimate_tokens, get_context_length
+            from src.llm_core import llm_call_async
+            from src.endpoint_resolver import resolve_endpoint
+
+            if len(session.history) < 6:
+                return {"status": "ok", "message": "Not enough messages to compact"}
+
+            ctx_len = get_context_length(session.endpoint_url, session.model)
+            messages_before = session.get_context_messages()
+            used_before = estimate_tokens(messages_before)
+            pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
+            msg_count_before = len(session.history)
+
+            # Keep only last 4 messages, summarize the rest
+            keep_count = 4
+            older = session.history[:-keep_count]
+            recent = session.history[-keep_count:]
+
+            # Build text to summarize
+            convo_text = "\n".join(
+                f"{_message_role(m).upper()}: "
+                f"{_message_text(m)[:2000]}"
+                for m in older
+            )
+
+            # Use utility model if available
+            util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
+            compact_url = util_url or session.endpoint_url
+            compact_model = util_model or session.model
+            compact_headers = util_headers if util_url else session.headers
+
+            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
+            compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
+            sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
+            summary = await llm_call_async(
+                compact_url, compact_model,
+                [
+                    {"role": "system", "content": sys_prompt},
+                    {"role": "user", "content": convo_text},
+                ],
+                temperature=0.2, max_tokens=1024,
+                headers=compact_headers, timeout=30,
+            )
+
+            # Replace session history: summary as system message + recent messages
+            # System message holds the full summary for AI context
+            system_summary = ChatMessage(
+                role="system",
+                content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
+                metadata={"compacted": True, "hidden": True},
+            )
+            # Visible assistant message just shows stats
+            summary_msg = ChatMessage(
+                role="assistant",
+                content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
+                metadata={"compacted": True, "messages_removed": len(older)},
+            )
+            new_history = [system_summary, summary_msg] + list(recent)
+            session.history = new_history
+            session.message_count = len(session.history)
+            logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
+
+            # Update DB: delete old messages, insert summary
+            db = SessionLocal()
+            try:
+                db_msgs = db.query(DbChatMessage).filter(
+                    DbChatMessage.session_id == session_id
+                ).order_by(DbChatMessage.timestamp).all()
+
+                # Delete all but the last keep_count
+                for m in db_msgs[:-keep_count]:
+                    db.delete(m)
+
+                # Insert system summary (hidden, for AI context) and visible summary
+                import json as _json
+                import uuid
+                from datetime import datetime, timezone
+                now = datetime.now(timezone.utc)
+                db_sys_summary = DbChatMessage(
+                    id=str(uuid.uuid4()),
+                    session_id=session_id,
+                    role="system",
+                    content=system_summary.content,
+                    meta_data=_json.dumps(system_summary.metadata),
+                    timestamp=now,
+                )
+                db.add(db_sys_summary)
+                db_summary = DbChatMessage(
+                    id=str(uuid.uuid4()),
+                    session_id=session_id,
+                    role="assistant",
+                    content=summary_msg.content,
+                    meta_data=_json.dumps(summary_msg.metadata),
+                    timestamp=now,
+                )
+                db.add(db_summary)
+
+                # Update session record
+                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+                if db_session:
+                    db_session.message_count = len(session.history)
+                    db_session.updated_at = datetime.now(timezone.utc)
+                db.commit()
+            finally:
+                db.close()
+
+            session_manager.save_sessions()
+
+            used_after = estimate_tokens(session.get_context_messages())
+            pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
+
+            return {
+                "status": "ok",
+                "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
+                "before": pct_before,
+                "after": pct_after,
+            }
+
+        except Exception as e:
+            logger.error(f"Manual compact error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
+    return router

routes/hwfit_routes.py

@@ -191,7 +191,7 @@ def setup_hwfit_routes():
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -200,17 +200,11 @@ def setup_hwfit_routes():
         fresh=true bypasses the hardware-detection cache."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.fit import rank_models
-        from services.hwfit.models import get_models, model_catalog_path, refresh_dynamic_catalogs
+        from services.hwfit.models import get_models, model_catalog_path
         host, ssh_port = _validate_detection_target(host, ssh_port)
         system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
         if system.get("error"):
             return {"system": system, "models": [], "error": system["error"]}
-        catalog_refresh = None
-        if refresh_catalog:
-            try:
-                catalog_refresh = refresh_dynamic_catalogs(force=True)
-            except Exception as e:
-                catalog_refresh = {"error": str(e)}
         if not get_models():
             return {
                 "system": system,
@@ -310,10 +304,7 @@ def setup_hwfit_routes():
             rank_kwargs.pop("target_context", None)
             rank_kwargs.pop("fit_only", None)
         results = rank_models(system, **rank_kwargs)
-        payload = {"system": system, "models": results}
+        return {"system": system, "models": results}
-        if catalog_refresh is not None:
-            payload["catalog_refresh"] = catalog_refresh
-        return payload
 
     @router.get("/profiles")
     def get_serve_profiles(model: str = "", model_path: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):

routes/memory/__init__.py

@@ -1,5 +0,0 @@
-"""Memory route domain package (slice 2c, #4082/#4071).
-
-Contains memory_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/memory_routes.py re-exports from here.
-"""

routes/memory/memory_routes.py

@@ -1,552 +0,0 @@
-# routes/memory_routes.py
-from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
-from typing import Dict, Any, Optional, List
-import json
-import os
-import re
-import tempfile
-import time
-from datetime import datetime
-import logging
-
-# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
-# Strips one prefix per call so import-from-LLM-output doesn't leave the
-# numbering inside the saved memory text. Bullet markers (-, *, •) are
-# also peeled here for the same reason.
-_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
-
-
-def _strip_list_prefix(text: str) -> str:
-    if not text:
-        return text
-    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
-
-from services.memory import MemoryManager
-from core.session_manager import SessionManager
-from src.request_models import MemoryAddRequest
-from core.database import SessionLocal
-from src.llm_core import llm_call_async
-from services.memory.memory_extractor import audit_memories
-from src.auth_helpers import get_current_user, require_user
-from src.endpoint_resolver import resolve_endpoint
-from src.task_endpoint import resolve_task_endpoint
-from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
-
-logger = logging.getLogger(__name__)
-
-
-def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
-    """Set up memory-related routes."""
-    router = APIRouter(prefix="/api/memory", tags=["memory"])
-
-    def _owner(request: Request) -> Optional[str]:
-        return get_current_user(request)
-
-    def _assert_session_owner(session_obj, user):
-        """SECURITY: 404 if the caller does not own this session.
-
-        SessionManager.get_session is NOT owner-scoped — it returns any
-        session by id. These routes accept a caller-supplied session id, so
-        without this gate a user could target another tenant's session and
-        leak their chat history, their session-scoped LLM credentials, or the
-        session title. Mirrors session_routes / webhook_routes ownership.
-        """
-        if user is not None and getattr(session_obj, "owner", None) != user:
-            raise HTTPException(404, "Session not found")
-
-    def _verify_memory_owner(memory: dict, user: Optional[str]):
-        """Raise 404 if user doesn't own this memory.
-
-        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
-        allowed any user to read/edit/delete memories with an empty/null owner
-        field, which leaked legacy data across the multi-user deploy.
-        """
-        if user is None:
-            return  # Auth disabled
-        if memory.get("owner") != user:
-            raise HTTPException(404, "Memory not found")
-
-    @router.post("/debug")
-    def debug_memory_relevance(request: Request, query: str = Form(...)):
-        """Debug which memories would be triggered for a query"""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
-
-        return {
-            "query": query,
-            "total_memories": len(memories),
-            "relevant_count": len(relevant),
-            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
-                                 for m in relevant]
-        }
-
-    @router.post("/add", response_model=Dict[str, Any])
-    async def api_add_memory(
-        request: Request,
-        memory_data: Optional[MemoryAddRequest] = None
-    ):
-        """Add a new memory entry with optional category, source, and session reference."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-        if memory_data is None:
-            form = await request.form()
-            memory_data = MemoryAddRequest(
-                text=form.get("text"),
-                category=form.get("category", "fact"),
-                source=form.get("source", "user"),
-                session_id=form.get("session_id")
-            )
-
-        user = _owner(request)
-        text = (memory_data.text or "").strip()
-        if not text:
-            raise HTTPException(400, "empty memory")
-        user_mem = memory_manager.load(owner=user)
-        if memory_manager.find_duplicates(text, user_mem):
-            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
-
-        if memory_data.session_id:
-            try:
-                session_obj = session_manager.get_session(memory_data.session_id)
-            except KeyError:
-                raise HTTPException(404, "Session not found")
-            _assert_session_owner(session_obj, user)
-
-        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
-        if memory_data.session_id:
-            new_entry["session_id"] = memory_data.session_id
-        all_mem = memory_manager.load_all()
-        all_mem.append(new_entry)
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.add(new_entry["id"], text)
-        try:
-            from src.event_bus import fire_event
-            fire_event("memory_added", user)
-        except Exception:
-            logger.debug("memory_added event dispatch failed", exc_info=True)
-        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
-
-    @router.get("")
-    def api_get_memory(request: Request):
-        """Return all memory entries with their metadata."""
-        user = _owner(request)
-        return {"memory": memory_manager.load(owner=user)}
-
-    @router.post("/search")
-    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
-        """Search across all memories with optional filters."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-
-        if session_id:
-            memories = [m for m in memories if m.get("session_id") == session_id]
-
-        if category:
-            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
-
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
-
-        return {"memories": relevant, "total": len(relevant), "query": query}
-
-    @router.get("/timeline")
-    def memory_timeline(request: Request):
-        """Get memories in chronological order with source session information."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        results = []
-        for memory in sorted_memories:
-            if "timestamp" in memory:
-                try:
-                    dt = datetime.fromtimestamp(memory["timestamp"])
-                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
-                except (ValueError, OSError, OverflowError):
-                    memory["timestamp_str"] = "Unknown"
-            else:
-                memory["timestamp_str"] = "Unknown"
-
-            session_id = memory.get("session_id")
-            if session_id and session_id in session_manager.sessions:
-                try:
-                    session = session_manager.get_session(session_id)
-                    if session:
-                        _assert_session_owner(session, user)
-                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
-                except KeyError:
-                    memory["session_name"] = "Unknown"
-                except HTTPException as exc:
-                    if exc.status_code != 404:
-                        raise
-                    memory["session_name"] = "Unknown"
-            else:
-                memory["session_name"] = "Unknown"
-
-            results.append(memory)
-
-        return {"timeline": results, "total": len(results)}
-
-    @router.get("/by-session/{session_id}")
-    def get_memory_by_session(request: Request, session_id: str):
-        """Get all memories associated with a specific session."""
-        user = _owner(request)
-        try:
-            _session_obj = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session {session_id} not found")
-        _assert_session_owner(_session_obj, user)
-        memories = memory_manager.load(owner=user)
-        session_memories = [m for m in memories if m.get("session_id") == session_id]
-
-        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        try:
-            session = session_manager.get_session(session_id)
-            session_name = session.name if session else f"Session {session_id[:6]}"
-        except KeyError:
-            session_name = f"Session {session_id[:6]}"
-
-        for memory in session_memories:
-            memory["session_name"] = session_name
-
-        return {
-            "session_id": session_id,
-            "session_name": session_name,
-            "memory_count": len(session_memories),
-            "memories": session_memories
-        }
-
-    @router.post("/extract")
-    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
-        """Analyze a session's chat history and return memory suggestions."""
-        require_user(request)
-        try:
-            sess = session_manager.get_session(session)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        _assert_session_owner(sess, _owner(request))
-
-        system_msg = {
-            "role": "system",
-            "content": (
-                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
-                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
-                "might want to remember for future interactions. Return each piece of information as a JSON object "
-                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
-                "Only include information that is specific and likely to be useful later."
-            ),
-        }
-        messages = [system_msg] + sess.get_context_messages()
-
-        t_url, t_model, t_headers = resolve_task_endpoint(
-            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
-        )
-
-        try:
-            suggestion_text = await llm_call_async(
-                t_url,
-                t_model,
-                messages,
-                temperature=0.2,
-                max_tokens=500,
-                headers=t_headers,
-            )
-            try:
-                suggestions = json.loads(suggestion_text)
-                if isinstance(suggestions, list):
-                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
-                else:
-                    suggestions = []
-            except json.JSONDecodeError:
-                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
-
-            return {"suggestions": [s for s in suggestions if s]}
-        except Exception as e:
-            logger.error(f"LLM memory extraction failed (session {session}): {e}")
-            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
-            return {"suggestions": [item["text"] for item in fallback]}
-
-    @router.post("/audit")
-    async def api_audit_memories(request: Request, session: str = Form(None)):
-        """Deduplicate and consolidate memories via LLM.
-
-        Uses task/utility/default settings through the shared resolver, with
-        the active session as fallback when no task or utility model is set.
-        Returns before and after memory counts.
-        """
-        user = _owner(request)
-        fallback_url = fallback_model = None
-        fallback_headers = None
-        if session:
-            try:
-                sess = session_manager.get_session(session)
-                _assert_session_owner(sess, user)
-                fallback_url = sess.endpoint_url
-                fallback_model = sess.model
-                fallback_headers = sess.headers
-            except KeyError:
-                pass
-
-        endpoint_url, model, headers = resolve_task_endpoint(
-            fallback_url, fallback_model, fallback_headers, owner=user
-        )
-
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No default model configured — set one in Settings")
-
-        result = await audit_memories(
-            memory_manager,
-            memory_vector,
-            endpoint_url,
-            model,
-            headers,
-            owner=user,
-        )
-
-        if "error" in result and "before" not in result:
-            raise HTTPException(502, f"Audit failed: {result['error']}")
-
-        return {
-            "ok": "error" not in result,
-            "before": result.get("before", 0),
-            "after": result.get("after", 0),
-            "removed": result.get("before", 0) - result.get("after", 0),
-            # True when the audit skipped the LLM because nothing changed
-            # since the last tidy. Frontend already says "Already clean"
-            # for removed==0, so this is here for future use / debugging.
-            "already_tidy": bool(result.get("already_tidy")),
-        }
-
-    @router.post("/import")
-    async def import_memories_from_file(
-        request: Request,
-        session: str | None = Form(None),
-        file: UploadFile = File(...)
-    ):
-        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-
-        endpoint_url = None
-        model = None
-        headers = {}
-
-        user = _owner(request)
-
-        if session:
-            try:
-                sess = session_manager.get_session(session)
-                _assert_session_owner(sess, user)
-            except KeyError:
-                sess = None
-            except HTTPException as exc:
-                if exc.status_code != 404:
-                    raise
-                sess = None
-
-            if sess is None:
-                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
-                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
-            else:
-                endpoint_url, model, headers = resolve_task_endpoint(
-                    sess.endpoint_url, sess.model, sess.headers, owner=user
-                )
-        else:
-            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
-    
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
-
-        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
-        filename = file.filename or "upload"
-        _, ext = os.path.splitext(filename.lower())
-
-        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
-        if ext not in allowed:
-            raise HTTPException(400, f"Unsupported file type: {ext}")
-
-        # Extract text based on file type
-        if ext == ".pdf":
-            from src.document_processor import _process_pdf
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
-                tmp.write(content)
-                tmp_path = tmp.name
-            try:
-                text = _process_pdf(tmp_path, owner=_owner(request))
-            finally:
-                os.unlink(tmp_path)
-        else:
-            try:
-                text = content.decode("utf-8")
-            except UnicodeDecodeError:
-                from charset_normalizer import detect
-                encoding = (detect(content) or {}).get("encoding") or "utf-8"
-                text = content.decode(encoding, errors="replace")
-
-        if not text.strip():
-            return {"suggestions": [], "message": "No readable content found"}
-
-        # Fast path: a .json upload that already looks like a memories export
-        # (list of {text, category, ...} dicts, or list of strings) round-trips
-        # directly without spending an LLM call to re-extract its own output.
-        # Without this, re-importing a memories.json from another account
-        # ran the file through the extractor, which often re-emitted the
-        # entries as a numbered list (and the numbering leaked into the
-        # `text` field).
-        if ext == ".json":
-            try:
-                parsed = json.loads(text)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list) and parsed:
-                direct = []
-                for item in parsed:
-                    if isinstance(item, dict) and item.get("text"):
-                        direct.append({
-                            "text": _strip_list_prefix(str(item["text"])),
-                            "category": item.get("category") or "fact",
-                        })
-                    elif isinstance(item, str) and item.strip():
-                        direct.append({
-                            "text": _strip_list_prefix(item.strip()),
-                            "category": "fact",
-                        })
-                if direct:
-                    return {"suggestions": direct, "filename": filename}
-
-        # Truncate very long documents
-        if len(text) > 15000:
-            text = text[:15000] + "\n[Truncated]"
-
-        # Send to LLM for memory extraction
-        import_prompt = (
-            "You are a memory extraction assistant. The user uploaded a document. "
-            "Analyze the text below and extract specific, useful facts — things like "
-            "names, preferences, jobs, locations, relationships, opinions, projects, "
-            "goals, contacts, or any other personal details worth remembering.\n\n"
-            "Rules:\n"
-            "- Each fact should be a short, self-contained statement\n"
-            "- Do NOT extract generic knowledge\n"
-            "- Focus on personal, memorable information\n"
-            "- If there are no useful facts, return an empty array\n\n"
-            "Return a JSON array of objects with 'text' and 'category' fields.\n"
-            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
-            "Return ONLY valid JSON, no markdown fences."
-        )
-
-        try:
-            raw = await llm_call_async(
-                endpoint_url,
-                model,
-                [
-                    {"role": "system", "content": import_prompt},
-                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
-                ],
-                temperature=0.2,
-                max_tokens=2000,
-                headers=headers,
-            )
-
-            # Parse JSON
-            raw = raw.strip()
-            if raw.startswith("```"):
-                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-
-            suggestions = json.loads(raw)
-            if isinstance(suggestions, list):
-                normalized = []
-                for s in suggestions:
-                    if not s:
-                        continue
-                    if isinstance(s, dict):
-                        s = dict(s)
-                        if s.get("text"):
-                            s["text"] = _strip_list_prefix(str(s["text"]))
-                        normalized.append(s)
-                    else:
-                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
-                suggestions = normalized
-            else:
-                suggestions = []
-
-            return {"suggestions": suggestions, "filename": filename}
-
-        except json.JSONDecodeError:
-            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
-            # numbering the model added so saved memories don't keep the prefix.
-            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
-            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
-        except Exception as e:
-            logger.error(f"Memory import extraction failed: {e}")
-            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
-
-    @router.post("/{memory_id}/pin")
-    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
-        """Pin or unpin a memory. Pinned memories are always included in context."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["pinned"] = pinned
-                memory_manager.save(all_mem)
-                return {"ok": True, "pinned": pinned}
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
-    @router.get("/{memory_id}")
-    def get_memory_item(request: Request, memory_id: str):
-        """Get a specific memory item by ID."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        for memory in memories:
-            if memory["id"] == memory_id:
-                return {"memory": memory}
-
-        raise HTTPException(404, "Memory not found")
-
-    @router.put("/{memory_id}")
-    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
-        """Update an existing memory item with new text and optional category."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["text"] = text.strip()
-                if category:
-                    all_mem[i]["category"] = category
-                all_mem[i]["timestamp"] = int(time.time())
-
-                memory_manager.save(all_mem)
-                # Sync vector index (remove old, add updated)
-                if memory_vector and memory_vector.healthy:
-                    memory_vector.remove(memory_id)
-                    memory_vector.add(memory_id, text.strip())
-                return {"ok": True, "message": "Memory updated successfully"}
-
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    @router.delete("/{memory_id}")
-    def delete_memory(request: Request, memory_id: str):
-        """Delete a memory item by its ID."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-
-        # Find and verify ownership before deleting
-        target = next((m for m in all_mem if m["id"] == memory_id), None)
-        if not target:
-            raise HTTPException(404, f"Memory item {memory_id} not found")
-        _verify_memory_owner(target, user)
-
-        all_mem = [m for m in all_mem if m["id"] != memory_id]
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.remove(memory_id)
-        return {"ok": True, "message": "Memory deleted successfully"}
-
-    return router

routes/memory_routes.py

@@ -1,18 +1,552 @@
-"""Backward-compat shim — canonical location is routes/memory/memory_routes.py.
+# routes/memory_routes.py
+from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
+from typing import Dict, Any, Optional, List
+import json
+import os
+import re
+import tempfile
+import time
+from datetime import datetime
+import logging
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
-that ``import routes.memory_routes``, ``from routes.memory_routes import X``,
+# Strips one prefix per call so import-from-LLM-output doesn't leave the
-``importlib.import_module("routes.memory_routes")``, and
+# numbering inside the saved memory text. Bullet markers (-, *, •) are
-``monkeypatch.setattr(routes.memory_routes, "ATTR", ...)`` (used by
+# also peeled here for the same reason.
-test_memory_routes_session_owner.py and test_memory_owner_isolation.py via
+_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
-``import ... as mr`` + ``setattr(mr, ...)``) all operate on the *same* object
-the application actually uses. Keeps existing import paths working after
-slice 2c (#4082/#4071). Source-introspection tests read the canonical file
-by path.
-"""
 
-import sys as _sys
 
-from routes.memory import memory_routes as _canonical  # noqa: F401
+def _strip_list_prefix(text: str) -> str:
+    if not text:
+        return text
+    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
 
-_sys.modules[__name__] = _canonical
+from services.memory import MemoryManager
+from core.session_manager import SessionManager
+from src.request_models import MemoryAddRequest
+from core.database import SessionLocal
+from src.llm_core import llm_call_async
+from services.memory.memory_extractor import audit_memories
+from src.auth_helpers import get_current_user, require_user
+from src.endpoint_resolver import resolve_endpoint
+from src.task_endpoint import resolve_task_endpoint
+from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
+
+logger = logging.getLogger(__name__)
+
+
+def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
+    """Set up memory-related routes."""
+    router = APIRouter(prefix="/api/memory", tags=["memory"])
+
+    def _owner(request: Request) -> Optional[str]:
+        return get_current_user(request)
+
+    def _assert_session_owner(session_obj, user):
+        """SECURITY: 404 if the caller does not own this session.
+
+        SessionManager.get_session is NOT owner-scoped — it returns any
+        session by id. These routes accept a caller-supplied session id, so
+        without this gate a user could target another tenant's session and
+        leak their chat history, their session-scoped LLM credentials, or the
+        session title. Mirrors session_routes / webhook_routes ownership.
+        """
+        if user is not None and getattr(session_obj, "owner", None) != user:
+            raise HTTPException(404, "Session not found")
+
+    def _verify_memory_owner(memory: dict, user: Optional[str]):
+        """Raise 404 if user doesn't own this memory.
+
+        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
+        allowed any user to read/edit/delete memories with an empty/null owner
+        field, which leaked legacy data across the multi-user deploy.
+        """
+        if user is None:
+            return  # Auth disabled
+        if memory.get("owner") != user:
+            raise HTTPException(404, "Memory not found")
+
+    @router.post("/debug")
+    def debug_memory_relevance(request: Request, query: str = Form(...)):
+        """Debug which memories would be triggered for a query"""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
+
+        return {
+            "query": query,
+            "total_memories": len(memories),
+            "relevant_count": len(relevant),
+            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
+                                 for m in relevant]
+        }
+
+    @router.post("/add", response_model=Dict[str, Any])
+    async def api_add_memory(
+        request: Request,
+        memory_data: Optional[MemoryAddRequest] = None
+    ):
+        """Add a new memory entry with optional category, source, and session reference."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+        if memory_data is None:
+            form = await request.form()
+            memory_data = MemoryAddRequest(
+                text=form.get("text"),
+                category=form.get("category", "fact"),
+                source=form.get("source", "user"),
+                session_id=form.get("session_id")
+            )
+
+        user = _owner(request)
+        text = (memory_data.text or "").strip()
+        if not text:
+            raise HTTPException(400, "empty memory")
+        user_mem = memory_manager.load(owner=user)
+        if memory_manager.find_duplicates(text, user_mem):
+            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
+
+        if memory_data.session_id:
+            try:
+                session_obj = session_manager.get_session(memory_data.session_id)
+            except KeyError:
+                raise HTTPException(404, "Session not found")
+            _assert_session_owner(session_obj, user)
+
+        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
+        if memory_data.session_id:
+            new_entry["session_id"] = memory_data.session_id
+        all_mem = memory_manager.load_all()
+        all_mem.append(new_entry)
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.add(new_entry["id"], text)
+        try:
+            from src.event_bus import fire_event
+            fire_event("memory_added", user)
+        except Exception:
+            logger.debug("memory_added event dispatch failed", exc_info=True)
+        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
+
+    @router.get("")
+    def api_get_memory(request: Request):
+        """Return all memory entries with their metadata."""
+        user = _owner(request)
+        return {"memory": memory_manager.load(owner=user)}
+
+    @router.post("/search")
+    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
+        """Search across all memories with optional filters."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+
+        if session_id:
+            memories = [m for m in memories if m.get("session_id") == session_id]
+
+        if category:
+            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
+
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+
+        return {"memories": relevant, "total": len(relevant), "query": query}
+
+    @router.get("/timeline")
+    def memory_timeline(request: Request):
+        """Get memories in chronological order with source session information."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        results = []
+        for memory in sorted_memories:
+            if "timestamp" in memory:
+                try:
+                    dt = datetime.fromtimestamp(memory["timestamp"])
+                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
+                except (ValueError, OSError, OverflowError):
+                    memory["timestamp_str"] = "Unknown"
+            else:
+                memory["timestamp_str"] = "Unknown"
+
+            session_id = memory.get("session_id")
+            if session_id and session_id in session_manager.sessions:
+                try:
+                    session = session_manager.get_session(session_id)
+                    if session:
+                        _assert_session_owner(session, user)
+                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
+                except KeyError:
+                    memory["session_name"] = "Unknown"
+                except HTTPException as exc:
+                    if exc.status_code != 404:
+                        raise
+                    memory["session_name"] = "Unknown"
+            else:
+                memory["session_name"] = "Unknown"
+
+            results.append(memory)
+
+        return {"timeline": results, "total": len(results)}
+
+    @router.get("/by-session/{session_id}")
+    def get_memory_by_session(request: Request, session_id: str):
+        """Get all memories associated with a specific session."""
+        user = _owner(request)
+        try:
+            _session_obj = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, f"Session {session_id} not found")
+        _assert_session_owner(_session_obj, user)
+        memories = memory_manager.load(owner=user)
+        session_memories = [m for m in memories if m.get("session_id") == session_id]
+
+        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        try:
+            session = session_manager.get_session(session_id)
+            session_name = session.name if session else f"Session {session_id[:6]}"
+        except KeyError:
+            session_name = f"Session {session_id[:6]}"
+
+        for memory in session_memories:
+            memory["session_name"] = session_name
+
+        return {
+            "session_id": session_id,
+            "session_name": session_name,
+            "memory_count": len(session_memories),
+            "memories": session_memories
+        }
+
+    @router.post("/extract")
+    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
+        """Analyze a session's chat history and return memory suggestions."""
+        require_user(request)
+        try:
+            sess = session_manager.get_session(session)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        _assert_session_owner(sess, _owner(request))
+
+        system_msg = {
+            "role": "system",
+            "content": (
+                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
+                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
+                "might want to remember for future interactions. Return each piece of information as a JSON object "
+                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
+                "Only include information that is specific and likely to be useful later."
+            ),
+        }
+        messages = [system_msg] + sess.get_context_messages()
+
+        t_url, t_model, t_headers = resolve_task_endpoint(
+            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
+        )
+
+        try:
+            suggestion_text = await llm_call_async(
+                t_url,
+                t_model,
+                messages,
+                temperature=0.2,
+                max_tokens=500,
+                headers=t_headers,
+            )
+            try:
+                suggestions = json.loads(suggestion_text)
+                if isinstance(suggestions, list):
+                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
+                else:
+                    suggestions = []
+            except json.JSONDecodeError:
+                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
+
+            return {"suggestions": [s for s in suggestions if s]}
+        except Exception as e:
+            logger.error(f"LLM memory extraction failed (session {session}): {e}")
+            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
+            return {"suggestions": [item["text"] for item in fallback]}
+
+    @router.post("/audit")
+    async def api_audit_memories(request: Request, session: str = Form(None)):
+        """Deduplicate and consolidate memories via LLM.
+
+        Uses task/utility/default settings through the shared resolver, with
+        the active session as fallback when no task or utility model is set.
+        Returns before and after memory counts.
+        """
+        user = _owner(request)
+        fallback_url = fallback_model = None
+        fallback_headers = None
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+                fallback_url = sess.endpoint_url
+                fallback_model = sess.model
+                fallback_headers = sess.headers
+            except KeyError:
+                pass
+
+        endpoint_url, model, headers = resolve_task_endpoint(
+            fallback_url, fallback_model, fallback_headers, owner=user
+        )
+
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No default model configured — set one in Settings")
+
+        result = await audit_memories(
+            memory_manager,
+            memory_vector,
+            endpoint_url,
+            model,
+            headers,
+            owner=user,
+        )
+
+        if "error" in result and "before" not in result:
+            raise HTTPException(502, f"Audit failed: {result['error']}")
+
+        return {
+            "ok": "error" not in result,
+            "before": result.get("before", 0),
+            "after": result.get("after", 0),
+            "removed": result.get("before", 0) - result.get("after", 0),
+            # True when the audit skipped the LLM because nothing changed
+            # since the last tidy. Frontend already says "Already clean"
+            # for removed==0, so this is here for future use / debugging.
+            "already_tidy": bool(result.get("already_tidy")),
+        }
+
+    @router.post("/import")
+    async def import_memories_from_file(
+        request: Request,
+        session: str | None = Form(None),
+        file: UploadFile = File(...)
+    ):
+        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+
+        endpoint_url = None
+        model = None
+        headers = {}
+
+        user = _owner(request)
+
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+            except KeyError:
+                sess = None
+            except HTTPException as exc:
+                if exc.status_code != 404:
+                    raise
+                sess = None
+
+            if sess is None:
+                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
+                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
+            else:
+                endpoint_url, model, headers = resolve_task_endpoint(
+                    sess.endpoint_url, sess.model, sess.headers, owner=user
+                )
+        else:
+            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
+    
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
+
+        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
+        filename = file.filename or "upload"
+        _, ext = os.path.splitext(filename.lower())
+
+        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
+        if ext not in allowed:
+            raise HTTPException(400, f"Unsupported file type: {ext}")
+
+        # Extract text based on file type
+        if ext == ".pdf":
+            from src.document_processor import _process_pdf
+            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+                tmp.write(content)
+                tmp_path = tmp.name
+            try:
+                text = _process_pdf(tmp_path, owner=_owner(request))
+            finally:
+                os.unlink(tmp_path)
+        else:
+            try:
+                text = content.decode("utf-8")
+            except UnicodeDecodeError:
+                from charset_normalizer import detect
+                encoding = (detect(content) or {}).get("encoding") or "utf-8"
+                text = content.decode(encoding, errors="replace")
+
+        if not text.strip():
+            return {"suggestions": [], "message": "No readable content found"}
+
+        # Fast path: a .json upload that already looks like a memories export
+        # (list of {text, category, ...} dicts, or list of strings) round-trips
+        # directly without spending an LLM call to re-extract its own output.
+        # Without this, re-importing a memories.json from another account
+        # ran the file through the extractor, which often re-emitted the
+        # entries as a numbered list (and the numbering leaked into the
+        # `text` field).
+        if ext == ".json":
+            try:
+                parsed = json.loads(text)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list) and parsed:
+                direct = []
+                for item in parsed:
+                    if isinstance(item, dict) and item.get("text"):
+                        direct.append({
+                            "text": _strip_list_prefix(str(item["text"])),
+                            "category": item.get("category") or "fact",
+                        })
+                    elif isinstance(item, str) and item.strip():
+                        direct.append({
+                            "text": _strip_list_prefix(item.strip()),
+                            "category": "fact",
+                        })
+                if direct:
+                    return {"suggestions": direct, "filename": filename}
+
+        # Truncate very long documents
+        if len(text) > 15000:
+            text = text[:15000] + "\n[Truncated]"
+
+        # Send to LLM for memory extraction
+        import_prompt = (
+            "You are a memory extraction assistant. The user uploaded a document. "
+            "Analyze the text below and extract specific, useful facts — things like "
+            "names, preferences, jobs, locations, relationships, opinions, projects, "
+            "goals, contacts, or any other personal details worth remembering.\n\n"
+            "Rules:\n"
+            "- Each fact should be a short, self-contained statement\n"
+            "- Do NOT extract generic knowledge\n"
+            "- Focus on personal, memorable information\n"
+            "- If there are no useful facts, return an empty array\n\n"
+            "Return a JSON array of objects with 'text' and 'category' fields.\n"
+            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
+            "Return ONLY valid JSON, no markdown fences."
+        )
+
+        try:
+            raw = await llm_call_async(
+                endpoint_url,
+                model,
+                [
+                    {"role": "system", "content": import_prompt},
+                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
+                ],
+                temperature=0.2,
+                max_tokens=2000,
+                headers=headers,
+            )
+
+            # Parse JSON
+            raw = raw.strip()
+            if raw.startswith("```"):
+                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+
+            suggestions = json.loads(raw)
+            if isinstance(suggestions, list):
+                normalized = []
+                for s in suggestions:
+                    if not s:
+                        continue
+                    if isinstance(s, dict):
+                        s = dict(s)
+                        if s.get("text"):
+                            s["text"] = _strip_list_prefix(str(s["text"]))
+                        normalized.append(s)
+                    else:
+                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
+                suggestions = normalized
+            else:
+                suggestions = []
+
+            return {"suggestions": suggestions, "filename": filename}
+
+        except json.JSONDecodeError:
+            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
+            # numbering the model added so saved memories don't keep the prefix.
+            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
+            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
+        except Exception as e:
+            logger.error(f"Memory import extraction failed: {e}")
+            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
+
+    @router.post("/{memory_id}/pin")
+    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
+        """Pin or unpin a memory. Pinned memories are always included in context."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["pinned"] = pinned
+                memory_manager.save(all_mem)
+                return {"ok": True, "pinned": pinned}
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
+    @router.get("/{memory_id}")
+    def get_memory_item(request: Request, memory_id: str):
+        """Get a specific memory item by ID."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        for memory in memories:
+            if memory["id"] == memory_id:
+                return {"memory": memory}
+
+        raise HTTPException(404, "Memory not found")
+
+    @router.put("/{memory_id}")
+    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
+        """Update an existing memory item with new text and optional category."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["text"] = text.strip()
+                if category:
+                    all_mem[i]["category"] = category
+                all_mem[i]["timestamp"] = int(time.time())
+
+                memory_manager.save(all_mem)
+                # Sync vector index (remove old, add updated)
+                if memory_vector and memory_vector.healthy:
+                    memory_vector.remove(memory_id)
+                    memory_vector.add(memory_id, text.strip())
+                return {"ok": True, "message": "Memory updated successfully"}
+
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    @router.delete("/{memory_id}")
+    def delete_memory(request: Request, memory_id: str):
+        """Delete a memory item by its ID."""
+        user = _owner(request)
+        all_mem = memory_manager.load_all()
+
+        # Find and verify ownership before deleting
+        target = next((m for m in all_mem if m["id"] == memory_id), None)
+        if not target:
+            raise HTTPException(404, f"Memory item {memory_id} not found")
+        _verify_memory_owner(target, user)
+
+        all_mem = [m for m in all_mem if m["id"] != memory_id]
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.remove(memory_id)
+        return {"ok": True, "message": "Memory deleted successfully"}
+
+    return router

routes/model_routes.py

@@ -19,7 +19,6 @@ from fastapi.responses import StreamingResponse
 from core.database import SessionLocal, ModelEndpoint, Session as DbSession
 from core.log_safety import redact_url as _redact_url_for_log
 from core.middleware import require_admin
-from src.constants import COOKBOOK_STATE_FILE
 from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
 from src.tls_overrides import llm_verify
 from src.settings import load_settings as _load_settings, save_settings as _save_settings
@@ -113,67 +112,6 @@ def _clear_endpoint_settings_for_endpoint(settings: dict, ep_id: str, *, include
     return cleared
 
 
-_COOKBOOK_ACTIVE_SERVE_STATUSES = {
-    "starting", "loading", "ready", "running", "restarting",
-}
-
-
-def _active_cookbook_endpoint_ids() -> set[str]:
-    """Endpoint IDs owned by active Cookbook serve tasks.
-
-    Cookbook auto-registers endpoints with ids like ``local-*``. Those rows are
-    managed lifecycle state, not durable user configuration. If a tmux stream is
-    stopped or an old task lingers, the row must stop participating in model
-    selection and defaults.
-    """
-    try:
-        if not os.path.exists(COOKBOOK_STATE_FILE):
-            return set()
-        with open(COOKBOOK_STATE_FILE, "r", encoding="utf-8") as fh:
-            raw = fh.read()
-        state = json.loads(raw)
-    except Exception:
-        return set()
-    out: set[str] = set()
-    for task in state.get("tasks") or []:
-        if not isinstance(task, dict) or task.get("type") != "serve":
-            continue
-        if str(task.get("status") or "").lower() not in _COOKBOOK_ACTIVE_SERVE_STATUSES:
-            continue
-        ep_id = task.get("_endpointId") or task.get("endpointId") or task.get("endpoint_id")
-        if ep_id:
-            out.add(str(ep_id))
-    return out
-
-
-def _disable_stale_cookbook_local_endpoints(db) -> int:
-    """Disable enabled cookbook endpoints whose serve task is no longer active."""
-    active_ids = _active_cookbook_endpoint_ids()
-    if not active_ids:
-        return 0
-    stale = (
-        db.query(ModelEndpoint)
-        .filter(ModelEndpoint.is_enabled == True)  # noqa: E712
-        .filter(ModelEndpoint.id.like("local-%"))
-        .filter(~ModelEndpoint.id.in_(active_ids))
-        .all()
-    )
-    if not stale:
-        return 0
-    settings = _load_settings()
-    touched_settings = False
-    for ep in stale:
-        ep.is_enabled = False
-        ep.model_refresh_mode = "disabled"
-        if _clear_endpoint_settings_for_endpoint(settings, ep.id):
-            touched_settings = True
-        logger.info("Disabled stale Cookbook endpoint %s (%s @ %s)", ep.id, ep.name, ep.base_url)
-    if touched_settings:
-        _save_settings(settings)
-    db.commit()
-    return len(stale)
-
-
 def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     """Remove endpoint references from scoped or legacy-flat user preferences."""
     if not isinstance(all_prefs, dict):
@@ -187,24 +125,7 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     return cleared_users
 
 
-def _endpoint_visible_model_ids(ep: Any) -> List[str]:
+def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool:
-    """Known visible model ids for an endpoint, including pinned/manual ids."""
-    if ep is None:
-        return []
-    return _visible_models(
-        getattr(ep, "cached_models", None),
-        getattr(ep, "hidden_models", None),
-        getattr(ep, "pinned_models", None),
-    )
-
-
-def _default_endpoint_needs_assignment(
-    current_default_id: str,
-    enabled_endpoint_ids,
-    *,
-    current_default_endpoint: Any = None,
-    current_default_model: str = "",
-) -> bool:
     """Whether the global default chat endpoint should be (re)assigned.
 
     True when nothing is configured yet, or the configured default no longer
@@ -216,14 +137,7 @@ def _default_endpoint_needs_assignment(
     """
     if not current_default_id:
         return True
-    if current_default_id not in enabled_endpoint_ids:
+    return current_default_id not in enabled_endpoint_ids
-        return True
-    if current_default_endpoint is None:
-        return False
-    if not (current_default_model or "").strip():
-        return True
-    visible = _endpoint_visible_model_ids(current_default_endpoint)
-    return bool(visible and current_default_model not in visible)
 
 
 # Loopback hosts a user might type for a local model server (LM Studio,
@@ -1152,36 +1066,6 @@ def _merge_model_ids(*lists):
     return out
 
 
-def _is_mlx_deepseek_v4_repo_id(model_id: str) -> bool:
-    m = str(model_id or "").lower()
-    return "mlx-community/deepseek-v4" in m
-
-
-def _is_mlx_deepseek_v4_shim_id(model_id: str) -> bool:
-    m = str(model_id or "").lower()
-    return "/.cache/odysseus/mlx-shims/deepseek-v4" in m
-
-
-def _filter_mlx_deepseek_v4_repo_when_shimmed(model_ids):
-    """Hide the broken MLX repo id when a launch-specific shim id is available.
-
-    mlx_lm.server may advertise the original HF repo id even though generation
-    only works through Odysseus' sanitized local shim. Keep the shim as the
-    submitted model id and remove the raw repo id from the picker/default list.
-    """
-    ids = list(model_ids or [])
-    has_shim = any(_is_mlx_deepseek_v4_shim_id(m) for m in ids)
-    if not has_shim:
-        return ids
-    return [m for m in ids if not _is_mlx_deepseek_v4_repo_id(m)]
-
-
-def _model_display_name(model_id: str) -> str:
-    if _is_mlx_deepseek_v4_shim_id(model_id):
-        return str(model_id or "").rstrip("/").split("/")[-1] or "DeepSeek-V4-Flash-4bit"
-    return str(model_id or "").split("/")[-1]
-
-
 def _visible_models(cached_models, hidden_models, pinned_models=None):
     """Merge cached + pinned model IDs, then filter out hidden ones.
 
@@ -1195,7 +1079,6 @@ def _visible_models(cached_models, hidden_models, pinned_models=None):
         _normalize_model_ids(cached_models),
         _normalize_model_ids(pinned_models),
     )
-    merged = _filter_mlx_deepseek_v4_repo_when_shimmed(merged)
     if not hidden_models:
         return merged
     hidden = set(_normalize_model_ids(hidden_models))
@@ -1311,8 +1194,6 @@ def setup_model_routes(model_discovery):
                 db = SessionLocal()
                 changed = False
                 try:
-                    if _disable_stale_cookbook_local_endpoints(db):
-                        changed = True
                     endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
                     now = _time.time()
                     groups: Dict[str, Dict[str, Any]] = {}
@@ -1386,8 +1267,6 @@ def setup_model_routes(model_discovery):
 
         db = SessionLocal()
         try:
-            if _disable_stale_cookbook_local_endpoints(db):
-                _invalidate_models_cache()
             q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
             if owner and not is_admin:
                 # Regular users see: their own endpoints + null-owner
@@ -1427,9 +1306,9 @@ def setup_model_routes(model_discovery):
                     "port": 0,
                     "url": chat_url,
                     "models": curated,
-                    "models_display": [_model_display_name(mid) for mid in curated],
+                    "models_display": [mid.split("/")[-1] for mid in curated],
                     "models_extra": extra,
-                    "models_extra_display": [_model_display_name(mid) for mid in extra],
+                    "models_extra_display": [mid.split("/")[-1] for mid in extra],
                     "endpoint_id": ep.id,
                     "endpoint_name": ep.name,
                     "category": category,
@@ -1457,7 +1336,7 @@ def setup_model_routes(model_discovery):
         return {"hosts": [], "items": items}
 
     @router.get("/models")
-    def api_models(request: Request, refresh: bool = False, background: bool = False):
+    def api_models(request: Request, refresh: bool = False):
         """Get available models — per-user (caller sees only their endpoints +
         legacy/shared null-owner rows). Cached per-user for 30s."""
         # Require auth; "" is the unconfigured single-user mode, treated as
@@ -1499,11 +1378,8 @@ def setup_model_routes(model_discovery):
             return cache_entry["data"]
         result = _fetch_models(owner=owner, is_admin=_is_admin)
         _models_cache[_cache_key] = {"data": result, "time": now}
-        # Kick off background refresh to update caches from live endpoints.
+        # Kick off background refresh to update caches from live endpoints
-        # Page boot can opt out with background=false so opening Odysseus does
+        _refresh_caches_bg(force=refresh)
-        # not start endpoint probes against slow/offline model servers.
-        if background or refresh:
-            _refresh_caches_bg(force=refresh)
         return result
 
     # Brief cache for local-probe results so picker-open doesn't hammer
@@ -1512,7 +1388,6 @@ def setup_model_routes(model_discovery):
     # within ~8s of the user noticing.
     _LOCAL_PROBE_TTL = 8.0
     _local_probe_cache: Dict[str, Any] = {"data": None, "time": 0.0}
-    _local_probe_inflight: Dict[str, Any] = {"task": None}
 
     @router.get("/model-endpoints/probe-local")
     async def probe_local_endpoints(request: Request):
@@ -1527,72 +1402,58 @@ def setup_model_routes(model_discovery):
                 (now - _local_probe_cache["time"]) < _LOCAL_PROBE_TTL):
             return _local_probe_cache["data"]
 
-        import asyncio as _asyncio
+        db = SessionLocal()
-        task = _local_probe_inflight.get("task")
-        if task is not None and not task.done():
-            return await task
-
-        async def _compute_local_probe() -> Dict[str, Any]:
-            db = SessionLocal()
-            try:
-                if _disable_stale_cookbook_local_endpoints(db):
-                    _invalidate_models_cache()
-                endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
-                local_eps = []
-                for ep in endpoints:
-                    base = _normalize_base(ep.base_url)
-                    kind = _effective_endpoint_kind(ep, base)
-                    if _classify_endpoint(base, kind) == "local":
-                        local_eps.append((ep.id, base, ep.api_key))
-            finally:
-                db.close()
-
-            grouped: Dict[str, Dict[str, Any]] = {}
-            for ep_id, base, api_key in local_eps:
-                key = _refresh_key(base, api_key)
-                grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
-
-            async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
-                t0 = _time.time()
-                try:
-                    # Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
-                    # local vLLM endpoints on Tailscale links where the model
-                    # server is still loading (Qwen3.5-122B takes 2–3 min to
-                    # warm); /v1/models can take 500–2500 ms on a busy box,
-                    # which pushed _ping_endpoint's full path-discovery sweep
-                    # past the cap and marked the row offline despite the
-                    # user actively chatting with it.
-                    ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
-                    lat = round((_time.time() - t0) * 1000)
-                    return {
-                        "alive": bool(ping.get("reachable")),
-                        "latency_ms": lat,
-                        "status_code": ping.get("status_code"),
-                        "error": ping.get("error"),
-                    }
-                except Exception as e:
-                    return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
-
-            results_list = await _asyncio.gather(
-                *[_probe_one(data) for data in grouped.values()],
-                return_exceptions=False,
-            )
-            results: Dict[str, Any] = {}
-            for data, r in zip(grouped.values(), results_list):
-                for eid in data["endpoint_ids"]:
-                    results[eid] = r
-
-            _local_probe_cache["data"] = results
-            _local_probe_cache["time"] = _time.time()
-            return results
-
-        task = _asyncio.create_task(_compute_local_probe())
-        _local_probe_inflight["task"] = task
         try:
-            return await task
+            endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
+            local_eps = []
+            for ep in endpoints:
+                base = _normalize_base(ep.base_url)
+                kind = _effective_endpoint_kind(ep, base)
+                if _classify_endpoint(base, kind) == "local":
+                    local_eps.append((ep.id, base, ep.api_key))
         finally:
-            if _local_probe_inflight.get("task") is task:
+            db.close()
-                _local_probe_inflight["task"] = None
+
+        grouped: Dict[str, Dict[str, Any]] = {}
+        for ep_id, base, api_key in local_eps:
+            key = _refresh_key(base, api_key)
+            grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
+
+        async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
+            t0 = _time.time()
+            try:
+                import asyncio as _asyncio
+                # Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
+                # local vLLM endpoints on Tailscale links where the model
+                # server is still loading (Qwen3.5-122B takes 2–3 min to
+                # warm); /v1/models can take 500–2500 ms on a busy box,
+                # which pushed _ping_endpoint's full path-discovery sweep
+                # past the cap and marked the row offline despite the
+                # user actively chatting with it.
+                ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
+                lat = round((_time.time() - t0) * 1000)
+                return {
+                    "alive": bool(ping.get("reachable")),
+                    "latency_ms": lat,
+                    "status_code": ping.get("status_code"),
+                    "error": ping.get("error"),
+                }
+            except Exception as e:
+                return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
+
+        import asyncio as _asyncio
+        results_list = await _asyncio.gather(
+            *[_probe_one(data) for data in grouped.values()],
+            return_exceptions=False,
+        )
+        results: Dict[str, Any] = {}
+        for data, r in zip(grouped.values(), results_list):
+            for eid in data["endpoint_ids"]:
+                results[eid] = r
+
+        _local_probe_cache["data"] = results
+        _local_probe_cache["time"] = now
+        return results
 
     @router.get("/ping")
     def ping_endpoints(request: Request):
@@ -1775,8 +1636,6 @@ def setup_model_routes(model_discovery):
         require_admin(request)
         db = SessionLocal()
         try:
-            if _disable_stale_cookbook_local_endpoints(db):
-                _invalidate_models_cache()
             rows = db.query(ModelEndpoint).order_by(ModelEndpoint.created_at).all()
             results = []
             for r in rows:
@@ -1784,11 +1643,67 @@ def setup_model_routes(model_discovery):
                 hidden = _hidden_model_ids(r)
                 pinned = _normalize_model_ids(getattr(r, "pinned_models", None))
                 visible = _visible_models(all_models, r.hidden_models, pinned)
-                # Keep the list route cache-only. It feeds Settings →
+                # Endpoint counts as reachable if it has any model — including
-                # Added Models and must render immediately; explicit
+                # admin-pinned IDs that a probe would never surface.
-                # Refresh/Probe endpoints do the network work.
+                status = "online" if (all_models or pinned) else "offline"
-                status = "online" if (all_models or pinned) else ("empty" if r.is_enabled else "offline")
                 ping = None
+                # When cached_models is empty, do a quick reachability probe.
+                # Bumped 1.0s → 3.5s because the user reported endpoints they
+                # were ACTIVELY chatting with showed "offline" — the previous
+                # 1s timeout was clipping live cloud endpoints (DeepSeek can
+                # take 1.5–2.5s on /v1/models when their region is under load,
+                # vLLM on a remote GPU box behind SSH can also push past 1s).
+                # 3.5s still keeps the picker render snappy in the common
+                # "everything's already cached" path because this branch only
+                # runs for endpoints with an empty cached_models.
+                if not all_models and not pinned and r.is_enabled:
+                    base_for_ping = _normalize_base(r.base_url)
+                    kind_for_ping = _effective_endpoint_kind(r, base_for_ping)
+                    ping_timeout = 10.0 if _classify_endpoint(base_for_ping, kind_for_ping) == "local" else 3.5
+                    ping = _ping_endpoint(r.base_url, r.api_key, timeout=ping_timeout)
+                    if ping.get("reachable"):
+                        status = "loading" if ping.get("loading") else "empty"
+                        if ping.get("loading"):
+                            base = _normalize_base(r.base_url)
+                            kind = _effective_endpoint_kind(r, base)
+                            results.append({
+                                "id": r.id,
+                                "name": r.name,
+                                "base_url": r.base_url,
+                                "has_key": bool(r.api_key),
+                                "api_key_fingerprint": _api_key_fingerprint(r.api_key),
+                                "is_enabled": r.is_enabled,
+                                "models": visible,
+                                "pinned_models": pinned,
+                                "hidden_count": len(hidden),
+                                "online": True,
+                                "status": status,
+                                "ping_error": (ping or {}).get("error") if ping else None,
+                                "model_type": getattr(r, "model_type", None) or "llm",
+                                "supports_tools": getattr(r, "supports_tools", None),
+                                "endpoint_kind": kind,
+                                "category": _classify_endpoint(base, kind),
+                                "model_refresh_mode": _endpoint_refresh_mode(r, kind),
+                                "model_refresh_interval": getattr(r, "model_refresh_interval", None),
+                                "model_refresh_timeout": getattr(r, "model_refresh_timeout", None),
+                            })
+                            continue
+                        # Best-effort: if the probe came back reachable, try
+                        # to populate cached_models in the background so the
+                        # NEXT picker load shows "online" instead of "empty".
+                        # Failure here is silent — we already returned the
+                        # "empty" status, and the existing background refresh
+                        # path will eventually fill it in too.
+                        try:
+                            probed = _probe_endpoint(r.base_url, r.api_key, timeout=max(5, int(ping_timeout)))
+                            if probed:
+                                r.cached_models = json.dumps(probed)
+                                db.commit()
+                                all_models = probed
+                                visible = _visible_models(all_models, r.hidden_models, pinned)
+                                status = "online"
+                        except Exception as _refill_err:
+                            logger.debug(f"opportunistic cached_models refill failed for {r.id}: {_refill_err!r}")
                 base = _normalize_base(r.base_url)
                 kind = _effective_endpoint_kind(r, base)
                 results.append({
@@ -1918,14 +1833,7 @@ def setup_model_routes(model_discovery):
                 if api_key.strip() and not existing.api_key:
                     existing.api_key = api_key.strip()
                     changed = True
-                # Keep duplicate endpoint registration cheap. This path is hit
+                if should_probe:
-                # by Cookbook/browser auto-register flows and can run while the
-                # user is sending a chat message. Probing a stale LAN endpoint
-                # here used to hold the request open for tens of seconds and
-                # contend with session creation, making "send" feel blocked.
-                # Explicit "require models" calls still probe; normal refresh
-                # belongs to /model-endpoints/{id}/models or /probe.
-                if require_model_list:
                     probed_models = _probe_endpoint(
                         base_url,
                         (api_key.strip() or existing.api_key or None),
@@ -2012,18 +1920,7 @@ def setup_model_routes(model_discovery):
                     ModelEndpoint.is_enabled == True  # noqa: E712
                 ).all()
             }
-            current_default_id = settings.get("default_endpoint_id") or ""
+            if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids):
-            current_default_ep = None
-            if current_default_id:
-                current_default_ep = db.query(ModelEndpoint).filter(
-                    ModelEndpoint.id == current_default_id
-                ).first()
-            if _default_endpoint_needs_assignment(
-                current_default_id,
-                enabled_ids,
-                current_default_endpoint=current_default_ep,
-                current_default_model=settings.get("default_model") or "",
-            ):
                 from src.endpoint_resolver import _first_chat_model
                 settings["default_endpoint_id"] = ep.id
                 settings["default_model"] = _first_chat_model(model_ids) or ""

routes/note_routes.py

@@ -483,22 +483,11 @@ async def dispatch_reminder(
                 api_key = intg.get("api_key", "")
                 if api_key:
                     hdrs["Authorization"] = f"Bearer {api_key}"
-                # SSRF guard — same check (and env knob) as the webhook branch
+                async with httpx.AsyncClient(timeout=10.0) as client:
-                # above: link-local / metadata addresses are always rejected;
+                    resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
-                # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
+                    ntfy_sent = resp.is_success
-                # so a ntfy base_url can't be pointed at internal services.
+                    if not ntfy_sent:
-                import os as _os
+                        ntfy_error = f"ntfy returned HTTP {resp.status_code}"
-                from src.url_safety import check_outbound_url as _chk
-                _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
-                _ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
-                if not _ok:
-                    ntfy_error = f"ntfy URL rejected: {_reason}"
-                else:
-                    async with httpx.AsyncClient(timeout=10.0) as client:
-                        resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
-                        ntfy_sent = resp.is_success
-                        if not ntfy_sent:
-                            ntfy_error = f"ntfy returned HTTP {resp.status_code}"
             else:
                 ntfy_error = "No enabled ntfy integration"
         except Exception as e:

routes/research/__init__.py

@@ -1,5 +0,0 @@
-"""Research route domain package (slice 2b, #4082/#4071).
-
-Contains research_routes.py, migrated from the flat routes/ directory.
-Backward-compat shim at routes/research_routes.py re-exports from here.
-"""

routes/research/research_routes.py

@@ -1,783 +0,0 @@
-"""Research background task routes — /api/research/*."""
-
-import asyncio
-import json
-import logging
-import re
-import uuid
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
-
-from fastapi import APIRouter, HTTPException, Query, Request
-from fastapi.responses import HTMLResponse, StreamingResponse
-from pydantic import BaseModel, Field
-from core.middleware import INTERNAL_TOOL_USER
-from src.endpoint_resolver import resolve_endpoint
-from src.auth_helpers import _auth_disabled, get_current_user
-from core.auth import RESERVED_USERNAMES
-from src.constants import DEEP_RESEARCH_DIR
-
-_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
-
-
-def _validate_session_id(session_id: str) -> str:
-    if not _SESSION_ID_RE.fullmatch(session_id):
-        raise HTTPException(400, "Invalid session ID format")
-    return session_id
-
-
-def _research_storage_root() -> Path:
-    return Path(DEEP_RESEARCH_DIR).resolve()
-
-
-def _find_research_path(session_id: str) -> Path | None:
-    """Find a persisted research file without deriving its path from input."""
-    expected_name = f"{_validate_session_id(session_id)}.json"
-    root = _research_storage_root()
-    for stored_path in root.glob("*.json"):
-        if stored_path.name != expected_name:
-            continue
-        resolved = stored_path.resolve()
-        try:
-            resolved.relative_to(root)
-        except ValueError:
-            return None
-        if not resolved.is_file():
-            return None
-        return resolved
-    return None
-
-
-def _require_research_path(session_id: str) -> Path:
-    path = _find_research_path(session_id)
-    if path is None:
-        raise HTTPException(404, "Research not found")
-    return path
-
-
-def _find_owned_research_path(session_id: str, user: str) -> Path | None:
-    path = _find_research_path(session_id)
-    if path is None:
-        return None
-    try:
-        owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
-    except Exception:
-        return None
-    if owner != user:
-        return None
-    return path
-
-
-logger = logging.getLogger(__name__)
-
-# Model-name substrings that are NOT chat/generation models — research must
-# never pick these as its model. An OpenAI-style endpoint often lists
-# `text-embedding-ada-002` etc. first in its model list, which is why research
-# was failing with "Cannot reach model 'text-embedding-ada-002'".
-_NON_CHAT_MODEL = (
-    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
-    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
-)
-
-_RESEARCH_IMAGE_BLOCKLIST = {
-    "cdn.shopify.com/s/files/1/0179/4388/7926/files/icon.png",
-}
-
-
-def _is_research_icon_or_logo_url(url: str) -> bool:
-    path = url.lower().split("?")[0]
-    return any(token in path for token in (
-        "/logo", "logo_", "-logo", "favicon", "apple-touch-icon",
-        "sprite", "icon-", "_icon", "/icons/", "badge",
-    ))
-
-
-def _research_thumbnail(data: dict) -> str:
-    """Pick the same first visible image the visual report uses as hero."""
-    hidden = set(data.get("hidden_images") or [])
-    seen = set()
-
-    def usable(image: str) -> bool:
-        image = str(image or "").strip()
-        if not image or image in seen or image in hidden:
-            return False
-        if not image.startswith("https://"):
-            return False
-        if image.endswith((".svg", ".ico", ".gif")):
-            return False
-        if any(blocked in image for blocked in _RESEARCH_IMAGE_BLOCKLIST):
-            return False
-        if _is_research_icon_or_logo_url(image):
-            return False
-        return True
-
-    for source in data.get("sources") or []:
-        if not isinstance(source, dict):
-            continue
-        image = str(source.get("image") or source.get("og_image") or "").strip()
-        if usable(image):
-            seen.add(image)
-            return image
-    for finding in data.get("raw_findings") or data.get("findings") or []:
-        if not isinstance(finding, dict):
-            continue
-        image = str(finding.get("image") or finding.get("og_image") or "").strip()
-        if usable(image):
-            seen.add(image)
-            return image
-    return ""
-
-
-def _first_chat_model(models) -> str:
-    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
-    for m in (models or []):
-        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
-            return m
-    return (models[0] if models else "")
-
-
-def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
-    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
-    owner = owner or getattr(sess, "owner", None) or None
-    url, model, headers = resolve_endpoint(
-        "research",
-        fallback_url=sess.endpoint_url,
-        fallback_model=sess.model,
-        fallback_headers=sess.headers,
-        owner=owner,
-    )
-    return url, model, headers
-
-
-def _owned_enabled_endpoint(db, owner, endpoint_id=None):
-    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
-    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
-    None if nothing visible matches.
-
-    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
-    owner = private, "the model picker only shows the endpoint to that user") and
-    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
-    api_key + base_url into research_handler.start_research(llm_endpoint=,
-    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
-    via the bare first-enabled fallback — would let a research-privileged user
-    spend ANOTHER user's API key/quota and reach whatever internal base_url they
-    configured. Mirrors webhook_routes._first_enabled_endpoint and
-    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
-    legacy mode).
-    """
-    from src.database import ModelEndpoint
-    from src.auth_helpers import owner_filter
-    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
-    if endpoint_id:
-        q = q.filter(ModelEndpoint.id == endpoint_id)
-    return owner_filter(q, ModelEndpoint, owner).first()
-
-
-def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
-    """Resolve a ModelEndpoint row into (chat_url, model, headers).
-
-    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
-    panel-selected research endpoints. ChatGPT Subscription endpoints keep
-    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
-    """
-    from src.endpoint_resolver import (
-        build_chat_url,
-        build_headers,
-        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
-    )
-
-    try:
-        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
-    except Exception as e:
-        logger.warning("Could not resolve endpoint credentials for research: %s", e)
-        return None
-
-    ep_model = (model or "").strip()
-    if not ep_model:
-        try:
-            models = json.loads(ep.cached_models) if ep.cached_models else []
-            if models:
-                ep_model = _first_chat_model(models)
-        except Exception:
-            pass
-    if not ep_model:
-        return None
-    return build_chat_url(base), ep_model, build_headers(api_key, base)
-
-
-def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
-    router = APIRouter(tags=["research"])
-
-    def _require_user(request: Request) -> str:
-        """All research endpoints require an authenticated user. Research
-        data isn't owner-scoped in the on-disk JSON yet, so we at least
-        block anonymous access. Multi-tenant deploys should additionally
-        verify the session belongs to this user."""
-        user = get_current_user(request)
-        if not user:
-            if _auth_disabled():
-                return ""
-            raise HTTPException(401, "Not authenticated")
-        return user
-
-    def _owns_in_memory(session_id: str, user: str) -> bool:
-        """Ownership check for an in-flight (in-memory) research task.
-        Falls back to the on-disk JSON if the task has already finished."""
-        entry = research_handler._active_tasks.get(session_id)
-        if entry is not None:
-            return entry.get("owner", "") == user
-        # Task no longer in memory — check the persisted JSON.
-        try:
-            return _find_owned_research_path(session_id, user) is not None
-        except HTTPException:
-            return False
-
-    def _require_owned_or_active_research_path(session_id: str, user: str) -> Path | None:
-        """Validate ownership once and return the completed on-disk path.
-
-        Active running research has no completed disk path yet. Completed
-        tasks can remain in _active_tasks after persistence, so prefer their
-        owned disk path when available. Completed disk lookups still reuse the
-        path after the ownership gate.
-        """
-        entry = research_handler._active_tasks.get(session_id)
-        if entry is not None:
-            if entry.get("owner", "") != user:
-                raise HTTPException(404, "No research found for this session")
-            if entry.get("status") != "running":
-                path = _find_owned_research_path(session_id, user)
-                if path is not None:
-                    return path
-            return None
-
-        path = _find_owned_research_path(session_id, user)
-        if path is None:
-            raise HTTPException(404, "No research found for this session")
-        return path
-
-    @router.get("/api/research/active")
-    async def research_active(request: Request):
-        """List all currently active (running) research tasks."""
-        user = _require_user(request)
-        active = []
-        for sid, entry in research_handler._active_tasks.items():
-            # SECURITY: only show this user's running tasks.
-            if entry.get("owner", "") != user:
-                continue
-            if entry.get("status") == "running":
-                active.append({
-                    "session_id": sid,
-                    "query": entry.get("query", ""),
-                    "status": "running",
-                    "progress": entry.get("progress", {}),
-                    "started_at": entry.get("started_at", 0),
-                })
-        return {"active": active}
-
-    @router.get("/api/research/status/{session_id}")
-    async def research_status(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        status = research_handler.get_status(session_id)
-        if status is None:
-            raise HTTPException(404, "No research found for this session")
-        return status
-
-    @router.post("/api/research/cancel/{session_id}")
-    async def research_cancel(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        cancelled = research_handler.cancel_research(session_id)
-        return {"cancelled": cancelled}
-
-    @router.post("/api/research/result/{session_id}")
-    async def research_result(session_id: str, request: Request):
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research result available")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
-
-    def _assert_owns_research(session_id: str, user: str) -> None:
-        """404-not-403 ownership gate for a research session's on-disk JSON.
-        Use BEFORE returning any data or mutating the file."""
-        path = _require_research_path(session_id)
-        try:
-            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
-        except Exception:
-            raise HTTPException(404, "Research not found")
-        if owner != user:
-            raise HTTPException(404, "Research not found")
-
-    @router.get("/api/research/report/{session_id}")
-    async def research_report(session_id: str, request: Request):
-        """Serve the visual HTML report for a completed research session."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        logger.info(f"Visual report requested for session {session_id}")
-        try:
-            html_content = research_handler.get_report_html(session_id)
-        except Exception as e:
-            logger.error(f"Visual report generation error: {e}", exc_info=True)
-            raise HTTPException(500, f"Report generation failed: {e}")
-        if html_content is None:
-            logger.warning(f"No report data found for session {session_id}")
-            raise HTTPException(404, "No visual report available for this session")
-        return HTMLResponse(content=html_content)
-
-    class HideImageRequest(BaseModel):
-        url: str
-
-    @router.post("/api/research/{session_id}/hide-image")
-    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
-        """Mark an image URL as hidden for this research's visual report.
-        Persisted to the research JSON so subsequent /report renders skip it."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.hide_image(session_id, body.url)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.post("/api/research/{session_id}/unhide-images")
-    async def research_unhide_images(session_id: str, request: Request):
-        """Clear the hidden-images list for a research session."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.unhide_all_images(session_id)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.get("/api/research/library")
-    async def research_library(
-        request: Request,
-        search: Optional[str] = Query(None),
-        sort: str = Query("recent"),
-        limit: int = Query(50),
-        archived: bool = Query(False),
-    ):
-        user = _require_user(request)
-        """List all completed research for the Library panel."""
-        data_dir = Path(DEEP_RESEARCH_DIR)
-        items = []
-        for p in data_dir.glob("*.json"):
-            try:
-                d = json.loads(p.read_text(encoding="utf-8"))
-                # SECURITY: only show research belonging to this user. Legacy
-                # JSONs without an `owner` field are hidden — auth was the only
-                # gate before, so every user saw every other user's reports.
-                if d.get("owner") != user:
-                    continue
-                # Archived view shows ONLY archived reports; default hides them.
-                if bool(d.get("archived")) != archived:
-                    continue
-                query = d.get("query", "")
-                if search and search.lower() not in query.lower():
-                    continue
-                sources = d.get("sources", [])
-                items.append({
-                    "id": p.stem,
-                    "query": query,
-                    "category": d.get("category") or "",
-                    "source_count": len(sources),
-                    "status": d.get("status", "done"),
-                    "duration": d.get("stats", {}).get("Duration", ""),
-                    "rounds": d.get("stats", {}).get("Rounds", ""),
-                    "started_at": d.get("started_at", 0),
-                    "completed_at": d.get("completed_at", 0),
-                    "archived": bool(d.get("archived")),
-                    "thumbnail": _research_thumbnail(d),
-                })
-            except Exception:
-                continue
-
-        # Sort
-        if sort == "recent":
-            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
-        elif sort == "oldest":
-            items.sort(key=lambda x: x["completed_at"] or 0)
-        elif sort == "most-messages":
-            items.sort(key=lambda x: x["source_count"], reverse=True)
-        elif sort == "alpha":
-            items.sort(key=lambda x: x["query"].lower())
-
-        return {"research": items[:limit], "total": len(items)}
-
-    @router.get("/api/research/detail/{session_id}")
-    async def research_detail(session_id: str, request: Request):
-        """Return the full JSON for a single research result — sources,
-        summary, stats — used by the Library preview panel."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        path = _require_research_path(session_id)
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-        except Exception as e:
-            raise HTTPException(500, f"Failed to read research: {e}")
-        # SECURITY: 404 (not 403) so we don't leak that the report exists.
-        if data.get("owner") != user:
-            raise HTTPException(404, "Research not found")
-        return data
-
-    @router.post("/api/research/{session_id}/archive")
-    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
-        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        path = _require_research_path(session_id)
-        try:
-            data = json.loads(path.read_text(encoding="utf-8"))
-            if data.get("owner") != user:
-                raise HTTPException(404, "Research not found")
-            data["archived"] = bool(archived)
-            path.write_text(json.dumps(data), encoding="utf-8")
-        except HTTPException:
-            raise
-        except Exception as e:
-            raise HTTPException(500, f"Failed to update research: {e}")
-        return {"ok": True, "id": session_id, "archived": bool(archived)}
-
-    @router.delete("/api/research/{session_id}")
-    async def research_delete(session_id: str, request: Request):
-        """Delete a research result from disk."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        json_path = _find_research_path(session_id)
-        deleted = False
-        if json_path is not None:
-            # SECURITY: verify ownership before letting the caller delete it.
-            try:
-                data = json.loads(json_path.read_text(encoding="utf-8"))
-                if data.get("owner") != user:
-                    raise HTTPException(404, "Research not found")
-            except HTTPException:
-                raise
-            except Exception:
-                raise HTTPException(404, "Research not found")
-            json_path.unlink()
-            deleted = True
-        return {"deleted": deleted}
-
-    # ------------------------------------------------------------------
-    # Panel endpoints — launch research without a chat session
-    # ------------------------------------------------------------------
-
-    class ResearchStartRequest(BaseModel):
-        query: str
-        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
-        max_rounds: int = Field(default=0, ge=0, le=20)
-        search_provider: Optional[str] = None
-        endpoint_id: Optional[str] = None
-        model: Optional[str] = None
-        max_time: int = Field(default=300, ge=60, le=1800)
-        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
-        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
-        category: Optional[str] = None
-
-    @router.post("/api/research/start")
-    async def research_start(body: ResearchStartRequest, request: Request):
-        """Launch a research job from the dedicated panel."""
-        from src.auth_helpers import require_privilege
-        user = require_privilege(request, "can_use_research")
-        if user == INTERNAL_TOOL_USER:
-            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
-            if tool_owner and tool_owner not in RESERVED_USERNAMES:
-                auth_mgr = getattr(request.app.state, "auth_manager", None)
-                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
-                    try:
-                        privs = auth_mgr.get_privileges(tool_owner) or {}
-                        if not privs.get("can_use_research", True):
-                            raise HTTPException(403, f"Your account is not allowed to can use research.")
-                    except HTTPException:
-                        raise
-                    except Exception:
-                        pass
-                user = tool_owner
-        session_id = f"rp-{uuid.uuid4().hex[:12]}"
-
-        if body.endpoint_id:
-            from src.database import SessionLocal
-            db = SessionLocal()
-            try:
-                # Owner-scoped: never resolve another user's private endpoint
-                # (and its decrypted api_key / internal base_url). A scoped miss
-                # reads as 404 so the endpoint's existence isn't revealed.
-                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
-                if not ep:
-                    raise HTTPException(404, "Endpoint not found or disabled")
-                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
-                if not resolved:
-                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
-                ep_url, ep_model, ep_headers = resolved
-            finally:
-                db.close()
-        else:
-            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
-            # When neither research nor utility is configured, use the user's
-            # configured DEFAULT model (default_endpoint_id/default_model) rather
-            # than arbitrarily grabbing the first enabled endpoint's first model
-            # (which surfaced gpt-3.5). "Default" should mean the default model.
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
-            if not ep_url:
-                from src.database import SessionLocal
-                db = SessionLocal()
-                try:
-                    # Owner-scoped first-enabled fallback: the caller's own rows
-                    # + legacy null-owner shared rows only — never borrow another
-                    # user's private endpoint/api_key. Same fix as the
-                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
-                    ep = _owned_enabled_endpoint(db, user)
-                    if ep:
-                        resolved = _resolve_endpoint_runtime(ep, owner=user)
-                        if resolved:
-                            ep_url, ep_model, ep_headers = resolved
-                finally:
-                    db.close()
-            if not ep_url:
-                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
-            if body.model:
-                ep_model = body.model
-
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
-        research_handler.start_research(
-            session_id=session_id,
-            query=body.query,
-            llm_endpoint=ep_url,
-            llm_model=ep_model,
-            max_time=body.max_time,
-            llm_headers=ep_headers,
-            max_rounds=effective_max_rounds,
-            search_provider=body.search_provider or None,
-            category=body.category or None,
-            extraction_timeout=body.extraction_timeout,
-            extraction_concurrency=body.extraction_concurrency,
-            owner=user,
-        )
-        return {"session_id": session_id, "status": "running", "query": body.query}
-
-    @router.get("/api/research/stream/{session_id}")
-    async def research_stream(session_id: str, request: Request):
-        """SSE stream of research progress events."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        async def _generate():
-            last_progress = None
-            while True:
-                status = research_handler.get_status(session_id)
-                if status is None:
-                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
-                    return
-                st = status.get("status", "")
-                progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
-                if st != "running":
-                    final = {'status': st, 'final': True}
-                    task = research_handler._active_tasks.get(session_id, {})
-                    if st == "error" and task.get("result"):
-                        final['error'] = str(task["result"])[:500]
-                    yield f"data: {json.dumps(final)}\n\n"
-                    return
-                await asyncio.sleep(1.5)
-
-        return StreamingResponse(
-            _generate(),
-            media_type="text/event-stream",
-            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
-        )
-
-    @router.post("/api/research/result-peek/{session_id}")
-    async def research_result_peek(session_id: str, request: Request):
-        """Get research result without clearing it (for panel use)."""
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        owned_disk_path = _require_owned_or_active_research_path(session_id, user)
-        result = research_handler.get_result(session_id)
-        if result is None:
-            p = owned_disk_path
-            if p is not None:
-                d = json.loads(p.read_text(encoding="utf-8"))
-                return {
-                    "result": d.get("result", ""),
-                    "sources": d.get("sources", []),
-                    "raw_findings": d.get("raw_findings", []),
-                    "category": d.get("category") or "",
-                }
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
-
-    @router.post("/api/research/spinoff/{session_id}")
-    async def research_spinoff(session_id: str, request: Request):
-        """Create a new chat session pre-seeded with this research as context.
-
-        Reads the persisted research result + sources for `session_id`, creates
-        a fresh session (inheriting endpoint/model/headers from the source
-        session if available, otherwise from the resolved chat endpoint), and
-        injects a single system message containing the report and sources so
-        the user can ask follow-up questions in a clean conversation.
-        """
-        user = _require_user(request)
-        _validate_session_id(session_id)
-        # SECURITY: gate on ownership before reading the persisted research —
-        # otherwise any authenticated user could spin off (and thereby read)
-        # another user's report by guessing its session ID. Mirrors every other
-        # endpoint in this file (see result_peek above).
-        owned_disk_path = _require_owned_or_active_research_path(session_id, user)
-        if session_manager is None:
-            raise HTTPException(500, "session_manager not configured")
-
-        # Load research data — prefer in-memory result, fall back to disk
-        result = research_handler.get_result(session_id)
-        sources = research_handler.get_sources(session_id) or []
-        query = ""
-
-        path = owned_disk_path
-        if path is not None:
-            try:
-                disk = json.loads(path.read_text(encoding="utf-8"))
-                if not result:
-                    result = disk.get("result")
-                if not sources:
-                    sources = disk.get("sources", []) or []
-                query = disk.get("query", "") or ""
-            except Exception as e:
-                logger.warning(f"Could not read research JSON for spinoff: {e}")
-
-        if not result:
-            raise HTTPException(404, "No research result available for this session")
-
-        # Inherit endpoint/model/headers from the source session when possible.
-        # For panel-launched research (rp-* IDs), there is no chat session, so
-        # fall back through the same chain as /api/research/start: research →
-        # utility → first enabled endpoint in the DB.
-        ep_url, ep_model, ep_headers = "", "", {}
-        try:
-            src_sess = session_manager.get_session(session_id)
-            ep_url = src_sess.endpoint_url or ""
-            ep_model = src_sess.model or ""
-            ep_headers = dict(src_sess.headers or {})
-        except KeyError:
-            pass
-
-        def _merge(r_url, r_model, r_headers):
-            nonlocal ep_url, ep_model, ep_headers
-            if not ep_url and r_url:
-                ep_url = r_url
-            if not ep_model and r_model:
-                ep_model = r_model
-            if not ep_headers and r_headers:
-                ep_headers = dict(r_headers)
-
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("chat", owner=user))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("research", owner=user))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("utility", owner=user))
-        if not ep_url or not ep_model:
-            # Last resort: this user's enabled endpoint, plus legacy shared rows.
-            from src.database import SessionLocal
-            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
-            db = SessionLocal()
-            try:
-                ep = _owned_enabled_endpoint(db, user)
-                if ep:
-                    base = normalize_base(ep.base_url)
-                    fallback_url = build_chat_url(base)
-                    fallback_headers = build_headers(ep.api_key, base)
-                    fallback_model = ""
-                    if ep.cached_models:
-                        try:
-                            models = json.loads(ep.cached_models)
-                            if models:
-                                fallback_model = _first_chat_model(models)
-                        except Exception:
-                            pass
-                    _merge(fallback_url, fallback_model, fallback_headers)
-            finally:
-                db.close()
-
-        if not ep_url or not ep_model:
-            raise HTTPException(400, "No endpoint configured — add one in Settings first")
-
-        # Create new session
-        new_sid = str(uuid.uuid4())
-
-        title_query = (query or "research").strip()
-        if len(title_query) > 60:
-            title_query = title_query[:57] + "…"
-        new_name = f"Follow-up: {title_query}"
-
-        new_sess = session_manager.create_session(
-            session_id=new_sid,
-            name=new_name,
-            endpoint_url=ep_url,
-            model=ep_model,
-            rag=False,
-            owner=user,
-        )
-        if ep_headers:
-            new_sess.headers = ep_headers
-            session_manager.save_sessions()
-        try:
-            from src.event_bus import fire_event
-            fire_event("session_created", user)
-        except Exception:
-            logger.debug("session_created event dispatch failed", exc_info=True)
-
-        # Build the priming system message — report only, no sources injected.
-        # The user can open the visual report for source details; keeping sources
-        # out of the chat context saves tokens and avoids the AI fabricating
-        # citations.
-        date_str = datetime.utcnow().strftime("%Y-%m-%d")
-        primer = (
-            f"[Research context — {date_str}]\n\n"
-            f"The user previously ran a deep research investigation. Use the "
-            f"report below as your primary knowledge base when answering "
-            f"follow-up questions. If the user asks something not covered, "
-            f"say so plainly rather than guessing.\n\n"
-            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
-            f"=== REPORT ===\n{result}"
-        )
-
-        from core.models import ChatMessage
-        new_sess.add_message(ChatMessage(
-            role="system",
-            content=primer,
-            metadata={"research_spinoff_from": session_id},
-        ))
-        session_manager.save_sessions()
-
-        return {
-            "session_id": new_sid,
-            "name": new_name,
-            "source_count": len(sources),
-        }
-
-    return router

routes/research_routes.py

@@ -1,17 +1,678 @@
-"""Backward-compat shim — canonical location is routes/research/research_routes.py.
+"""Research background task routes — /api/research/*."""
 
-This module is replaced in ``sys.modules`` by the canonical module object so
+import asyncio
-that ``import routes.research_routes``, ``from routes.research_routes import X``,
+import json
-``importlib.import_module("routes.research_routes")``, and
+import logging
-``monkeypatch.setattr("routes.research_routes.ATTR", ...)`` (string-targeted
+import re
-patch used by ``test_research_owner_scope_routes.py``) all operate on the
+import uuid
-*same* object the application actually uses. Keeps existing import paths
+from datetime import datetime
-working after slice 2b (#4082/#4071). Source-introspection tests read the
+from pathlib import Path
-canonical file by path.
+from typing import Optional
-"""
 
-import sys as _sys
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import HTMLResponse, StreamingResponse
+from pydantic import BaseModel, Field
+from core.middleware import INTERNAL_TOOL_USER
+from src.endpoint_resolver import resolve_endpoint
+from src.auth_helpers import _auth_disabled, get_current_user
+from core.auth import RESERVED_USERNAMES
+from src.constants import DEEP_RESEARCH_DIR
 
-from routes.research import research_routes as _canonical  # noqa: F401
+_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
 
-_sys.modules[__name__] = _canonical
+logger = logging.getLogger(__name__)
+
+# Model-name substrings that are NOT chat/generation models — research must
+# never pick these as its model. An OpenAI-style endpoint often lists
+# `text-embedding-ada-002` etc. first in its model list, which is why research
+# was failing with "Cannot reach model 'text-embedding-ada-002'".
+_NON_CHAT_MODEL = (
+    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
+    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
+)
+
+
+def _first_chat_model(models) -> str:
+    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
+    for m in (models or []):
+        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
+            return m
+    return (models[0] if models else "")
+
+
+def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
+    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
+    owner = owner or getattr(sess, "owner", None) or None
+    url, model, headers = resolve_endpoint(
+        "research",
+        fallback_url=sess.endpoint_url,
+        fallback_model=sess.model,
+        fallback_headers=sess.headers,
+        owner=owner,
+    )
+    return url, model, headers
+
+
+def _owned_enabled_endpoint(db, owner, endpoint_id=None):
+    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
+    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
+    None if nothing visible matches.
+
+    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
+    owner = private, "the model picker only shows the endpoint to that user") and
+    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
+    api_key + base_url into research_handler.start_research(llm_endpoint=,
+    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
+    via the bare first-enabled fallback — would let a research-privileged user
+    spend ANOTHER user's API key/quota and reach whatever internal base_url they
+    configured. Mirrors webhook_routes._first_enabled_endpoint and
+    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
+    legacy mode).
+    """
+    from src.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if endpoint_id:
+        q = q.filter(ModelEndpoint.id == endpoint_id)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
+    """Resolve a ModelEndpoint row into (chat_url, model, headers).
+
+    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
+    panel-selected research endpoints. ChatGPT Subscription endpoints keep
+    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
+    """
+    from src.endpoint_resolver import (
+        build_chat_url,
+        build_headers,
+        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
+    )
+
+    try:
+        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
+    except Exception as e:
+        logger.warning("Could not resolve endpoint credentials for research: %s", e)
+        return None
+
+    ep_model = (model or "").strip()
+    if not ep_model:
+        try:
+            models = json.loads(ep.cached_models) if ep.cached_models else []
+            if models:
+                ep_model = _first_chat_model(models)
+        except Exception:
+            pass
+    if not ep_model:
+        return None
+    return build_chat_url(base), ep_model, build_headers(api_key, base)
+
+
+def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
+    router = APIRouter(tags=["research"])
+
+    def _require_user(request: Request) -> str:
+        """All research endpoints require an authenticated user. Research
+        data isn't owner-scoped in the on-disk JSON yet, so we at least
+        block anonymous access. Multi-tenant deploys should additionally
+        verify the session belongs to this user."""
+        user = get_current_user(request)
+        if not user:
+            if _auth_disabled():
+                return ""
+            raise HTTPException(401, "Not authenticated")
+        return user
+
+    def _validate_session_id(session_id: str) -> None:
+        if not _SESSION_ID_RE.fullmatch(session_id):
+            raise HTTPException(400, "Invalid session ID format")
+
+    def _owns_in_memory(session_id: str, user: str) -> bool:
+        """Ownership check for an in-flight (in-memory) research task.
+        Falls back to the on-disk JSON if the task has already finished."""
+        entry = research_handler._active_tasks.get(session_id)
+        if entry is not None:
+            return entry.get("owner", "") == user
+        # Task no longer in memory — check the persisted JSON.
+        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        if not path.exists():
+            return False
+        try:
+            return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
+        except Exception:
+            return False
+
+    @router.get("/api/research/active")
+    async def research_active(request: Request):
+        """List all currently active (running) research tasks."""
+        user = _require_user(request)
+        active = []
+        for sid, entry in research_handler._active_tasks.items():
+            # SECURITY: only show this user's running tasks.
+            if entry.get("owner", "") != user:
+                continue
+            if entry.get("status") == "running":
+                active.append({
+                    "session_id": sid,
+                    "query": entry.get("query", ""),
+                    "status": "running",
+                    "progress": entry.get("progress", {}),
+                    "started_at": entry.get("started_at", 0),
+                })
+        return {"active": active}
+
+    @router.get("/api/research/status/{session_id}")
+    async def research_status(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        status = research_handler.get_status(session_id)
+        if status is None:
+            raise HTTPException(404, "No research found for this session")
+        return status
+
+    @router.post("/api/research/cancel/{session_id}")
+    async def research_cancel(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        cancelled = research_handler.cancel_research(session_id)
+        return {"cancelled": cancelled}
+
+    @router.post("/api/research/result/{session_id}")
+    async def research_result(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research result available")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        research_handler.clear_result(session_id)
+        return {"result": result, "sources": sources, "raw_findings": raw_findings}
+
+    def _assert_owns_research(session_id: str, user: str) -> None:
+        """404-not-403 ownership gate for a research session's on-disk JSON.
+        Use BEFORE returning any data or mutating the file."""
+        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
+        except Exception:
+            raise HTTPException(404, "Research not found")
+        if owner != user:
+            raise HTTPException(404, "Research not found")
+
+    @router.get("/api/research/report/{session_id}")
+    async def research_report(session_id: str, request: Request):
+        """Serve the visual HTML report for a completed research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        logger.info(f"Visual report requested for session {session_id}")
+        try:
+            html_content = research_handler.get_report_html(session_id)
+        except Exception as e:
+            logger.error(f"Visual report generation error: {e}", exc_info=True)
+            raise HTTPException(500, f"Report generation failed: {e}")
+        if html_content is None:
+            logger.warning(f"No report data found for session {session_id}")
+            raise HTTPException(404, "No visual report available for this session")
+        return HTMLResponse(content=html_content)
+
+    class HideImageRequest(BaseModel):
+        url: str
+
+    @router.post("/api/research/{session_id}/hide-image")
+    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
+        """Mark an image URL as hidden for this research's visual report.
+        Persisted to the research JSON so subsequent /report renders skip it."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.hide_image(session_id, body.url)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.post("/api/research/{session_id}/unhide-images")
+    async def research_unhide_images(session_id: str, request: Request):
+        """Clear the hidden-images list for a research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.unhide_all_images(session_id)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.get("/api/research/library")
+    async def research_library(
+        request: Request,
+        search: Optional[str] = Query(None),
+        sort: str = Query("recent"),
+        limit: int = Query(50),
+        archived: bool = Query(False),
+    ):
+        user = _require_user(request)
+        """List all completed research for the Library panel."""
+        data_dir = Path(DEEP_RESEARCH_DIR)
+        items = []
+        for p in data_dir.glob("*.json"):
+            try:
+                d = json.loads(p.read_text(encoding="utf-8"))
+                # SECURITY: only show research belonging to this user. Legacy
+                # JSONs without an `owner` field are hidden — auth was the only
+                # gate before, so every user saw every other user's reports.
+                if d.get("owner") != user:
+                    continue
+                # Archived view shows ONLY archived reports; default hides them.
+                if bool(d.get("archived")) != archived:
+                    continue
+                query = d.get("query", "")
+                if search and search.lower() not in query.lower():
+                    continue
+                sources = d.get("sources", [])
+                items.append({
+                    "id": p.stem,
+                    "query": query,
+                    "category": d.get("category") or "",
+                    "source_count": len(sources),
+                    "status": d.get("status", "done"),
+                    "duration": d.get("stats", {}).get("Duration", ""),
+                    "rounds": d.get("stats", {}).get("Rounds", ""),
+                    "started_at": d.get("started_at", 0),
+                    "completed_at": d.get("completed_at", 0),
+                    "archived": bool(d.get("archived")),
+                })
+            except Exception:
+                continue
+
+        # Sort
+        if sort == "recent":
+            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
+        elif sort == "oldest":
+            items.sort(key=lambda x: x["completed_at"] or 0)
+        elif sort == "most-messages":
+            items.sort(key=lambda x: x["source_count"], reverse=True)
+        elif sort == "alpha":
+            items.sort(key=lambda x: x["query"].lower())
+
+        return {"research": items[:limit], "total": len(items)}
+
+    @router.get("/api/research/detail/{session_id}")
+    async def research_detail(session_id: str, request: Request):
+        """Return the full JSON for a single research result — sources,
+        summary, stats — used by the Library preview panel."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as e:
+            raise HTTPException(500, f"Failed to read research: {e}")
+        # SECURITY: 404 (not 403) so we don't leak that the report exists.
+        if data.get("owner") != user:
+            raise HTTPException(404, "Research not found")
+        return data
+
+    @router.post("/api/research/{session_id}/archive")
+    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
+        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        if not path.exists():
+            raise HTTPException(404, "Research not found")
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("owner") != user:
+                raise HTTPException(404, "Research not found")
+            data["archived"] = bool(archived)
+            path.write_text(json.dumps(data), encoding="utf-8")
+        except HTTPException:
+            raise
+        except Exception as e:
+            raise HTTPException(500, f"Failed to update research: {e}")
+        return {"ok": True, "id": session_id, "archived": bool(archived)}
+
+    @router.delete("/api/research/{session_id}")
+    async def research_delete(session_id: str, request: Request):
+        """Delete a research result from disk."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        data_dir = Path(DEEP_RESEARCH_DIR)
+        json_path = data_dir / f"{session_id}.json"
+        deleted = False
+        if json_path.exists():
+            # SECURITY: verify ownership before letting the caller delete it.
+            try:
+                data = json.loads(json_path.read_text(encoding="utf-8"))
+                if data.get("owner") != user:
+                    raise HTTPException(404, "Research not found")
+            except HTTPException:
+                raise
+            except Exception:
+                raise HTTPException(404, "Research not found")
+            json_path.unlink()
+            deleted = True
+        return {"deleted": deleted}
+
+    # ------------------------------------------------------------------
+    # Panel endpoints — launch research without a chat session
+    # ------------------------------------------------------------------
+
+    class ResearchStartRequest(BaseModel):
+        query: str
+        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
+        max_rounds: int = Field(default=0, ge=0, le=20)
+        search_provider: Optional[str] = None
+        endpoint_id: Optional[str] = None
+        model: Optional[str] = None
+        max_time: int = Field(default=300, ge=60, le=1800)
+        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
+        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
+        category: Optional[str] = None
+
+    @router.post("/api/research/start")
+    async def research_start(body: ResearchStartRequest, request: Request):
+        """Launch a research job from the dedicated panel."""
+        from src.auth_helpers import require_privilege
+        user = require_privilege(request, "can_use_research")
+        if user == INTERNAL_TOOL_USER:
+            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
+            if tool_owner and tool_owner not in RESERVED_USERNAMES:
+                auth_mgr = getattr(request.app.state, "auth_manager", None)
+                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
+                    try:
+                        privs = auth_mgr.get_privileges(tool_owner) or {}
+                        if not privs.get("can_use_research", True):
+                            raise HTTPException(403, f"Your account is not allowed to can use research.")
+                    except HTTPException:
+                        raise
+                    except Exception:
+                        pass
+                user = tool_owner
+        session_id = f"rp-{uuid.uuid4().hex[:12]}"
+
+        if body.endpoint_id:
+            from src.database import SessionLocal
+            db = SessionLocal()
+            try:
+                # Owner-scoped: never resolve another user's private endpoint
+                # (and its decrypted api_key / internal base_url). A scoped miss
+                # reads as 404 so the endpoint's existence isn't revealed.
+                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
+                if not ep:
+                    raise HTTPException(404, "Endpoint not found or disabled")
+                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
+                if not resolved:
+                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
+                ep_url, ep_model, ep_headers = resolved
+            finally:
+                db.close()
+        else:
+            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
+            # When neither research nor utility is configured, use the user's
+            # configured DEFAULT model (default_endpoint_id/default_model) rather
+            # than arbitrarily grabbing the first enabled endpoint's first model
+            # (which surfaced gpt-3.5). "Default" should mean the default model.
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
+            if not ep_url:
+                from src.database import SessionLocal
+                db = SessionLocal()
+                try:
+                    # Owner-scoped first-enabled fallback: the caller's own rows
+                    # + legacy null-owner shared rows only — never borrow another
+                    # user's private endpoint/api_key. Same fix as the
+                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
+                    ep = _owned_enabled_endpoint(db, user)
+                    if ep:
+                        resolved = _resolve_endpoint_runtime(ep, owner=user)
+                        if resolved:
+                            ep_url, ep_model, ep_headers = resolved
+                finally:
+                    db.close()
+            if not ep_url:
+                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
+            if body.model:
+                ep_model = body.model
+
+        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
+        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
+        research_handler.start_research(
+            session_id=session_id,
+            query=body.query,
+            llm_endpoint=ep_url,
+            llm_model=ep_model,
+            max_time=body.max_time,
+            llm_headers=ep_headers,
+            max_rounds=effective_max_rounds,
+            search_provider=body.search_provider or None,
+            category=body.category or None,
+            extraction_timeout=body.extraction_timeout,
+            extraction_concurrency=body.extraction_concurrency,
+            owner=user,
+        )
+        return {"session_id": session_id, "status": "running", "query": body.query}
+
+    @router.get("/api/research/stream/{session_id}")
+    async def research_stream(session_id: str, request: Request):
+        """SSE stream of research progress events."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        async def _generate():
+            last_progress = None
+            while True:
+                status = research_handler.get_status(session_id)
+                if status is None:
+                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
+                    return
+                st = status.get("status", "")
+                progress = status.get("progress", {})
+                if progress != last_progress:
+                    last_progress = progress
+                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
+                if st != "running":
+                    final = {'status': st, 'final': True}
+                    task = research_handler._active_tasks.get(session_id, {})
+                    if st == "error" and task.get("result"):
+                        final['error'] = str(task["result"])[:500]
+                    yield f"data: {json.dumps(final)}\n\n"
+                    return
+                await asyncio.sleep(1.5)
+
+        return StreamingResponse(
+            _generate(),
+            media_type="text/event-stream",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
+    @router.post("/api/research/result-peek/{session_id}")
+    async def research_result_peek(session_id: str, request: Request):
+        """Get research result without clearing it (for panel use)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+            if p.exists():
+                d = json.loads(p.read_text(encoding="utf-8"))
+                return {
+                    "result": d.get("result", ""),
+                    "sources": d.get("sources", []),
+                    "raw_findings": d.get("raw_findings", []),
+                    "category": d.get("category") or "",
+                }
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
+
+    @router.post("/api/research/spinoff/{session_id}")
+    async def research_spinoff(session_id: str, request: Request):
+        """Create a new chat session pre-seeded with this research as context.
+
+        Reads the persisted research result + sources for `session_id`, creates
+        a fresh session (inheriting endpoint/model/headers from the source
+        session if available, otherwise from the resolved chat endpoint), and
+        injects a single system message containing the report and sources so
+        the user can ask follow-up questions in a clean conversation.
+        """
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        # SECURITY: gate on ownership before reading the persisted research —
+        # otherwise any authenticated user could spin off (and thereby read)
+        # another user's report by guessing its session ID. Mirrors every other
+        # endpoint in this file (see result_peek above).
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        if session_manager is None:
+            raise HTTPException(500, "session_manager not configured")
+
+        # Load research data — prefer in-memory result, fall back to disk
+        result = research_handler.get_result(session_id)
+        sources = research_handler.get_sources(session_id) or []
+        query = ""
+
+        path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
+        if path.exists():
+            try:
+                disk = json.loads(path.read_text(encoding="utf-8"))
+                if not result:
+                    result = disk.get("result")
+                if not sources:
+                    sources = disk.get("sources", []) or []
+                query = disk.get("query", "") or ""
+            except Exception as e:
+                logger.warning(f"Could not read research JSON for spinoff: {e}")
+
+        if not result:
+            raise HTTPException(404, "No research result available for this session")
+
+        # Inherit endpoint/model/headers from the source session when possible.
+        # For panel-launched research (rp-* IDs), there is no chat session, so
+        # fall back through the same chain as /api/research/start: research →
+        # utility → first enabled endpoint in the DB.
+        ep_url, ep_model, ep_headers = "", "", {}
+        try:
+            src_sess = session_manager.get_session(session_id)
+            ep_url = src_sess.endpoint_url or ""
+            ep_model = src_sess.model or ""
+            ep_headers = dict(src_sess.headers or {})
+        except KeyError:
+            pass
+
+        def _merge(r_url, r_model, r_headers):
+            nonlocal ep_url, ep_model, ep_headers
+            if not ep_url and r_url:
+                ep_url = r_url
+            if not ep_model and r_model:
+                ep_model = r_model
+            if not ep_headers and r_headers:
+                ep_headers = dict(r_headers)
+
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("chat", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("research", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("utility", owner=user))
+        if not ep_url or not ep_model:
+            # Last resort: this user's enabled endpoint, plus legacy shared rows.
+            from src.database import SessionLocal
+            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
+            db = SessionLocal()
+            try:
+                ep = _owned_enabled_endpoint(db, user)
+                if ep:
+                    base = normalize_base(ep.base_url)
+                    fallback_url = build_chat_url(base)
+                    fallback_headers = build_headers(ep.api_key, base)
+                    fallback_model = ""
+                    if ep.cached_models:
+                        try:
+                            models = json.loads(ep.cached_models)
+                            if models:
+                                fallback_model = _first_chat_model(models)
+                        except Exception:
+                            pass
+                    _merge(fallback_url, fallback_model, fallback_headers)
+            finally:
+                db.close()
+
+        if not ep_url or not ep_model:
+            raise HTTPException(400, "No endpoint configured — add one in Settings first")
+
+        # Create new session
+        new_sid = str(uuid.uuid4())
+
+        title_query = (query or "research").strip()
+        if len(title_query) > 60:
+            title_query = title_query[:57] + "…"
+        new_name = f"Follow-up: {title_query}"
+
+        new_sess = session_manager.create_session(
+            session_id=new_sid,
+            name=new_name,
+            endpoint_url=ep_url,
+            model=ep_model,
+            rag=False,
+            owner=user,
+        )
+        if ep_headers:
+            new_sess.headers = ep_headers
+            session_manager.save_sessions()
+        try:
+            from src.event_bus import fire_event
+            fire_event("session_created", user)
+        except Exception:
+            logger.debug("session_created event dispatch failed", exc_info=True)
+
+        # Build the priming system message — report only, no sources injected.
+        # The user can open the visual report for source details; keeping sources
+        # out of the chat context saves tokens and avoids the AI fabricating
+        # citations.
+        date_str = datetime.utcnow().strftime("%Y-%m-%d")
+        primer = (
+            f"[Research context — {date_str}]\n\n"
+            f"The user previously ran a deep research investigation. Use the "
+            f"report below as your primary knowledge base when answering "
+            f"follow-up questions. If the user asks something not covered, "
+            f"say so plainly rather than guessing.\n\n"
+            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
+            f"=== REPORT ===\n{result}"
+        )
+
+        from core.models import ChatMessage
+        new_sess.add_message(ChatMessage(
+            role="system",
+            content=primer,
+            metadata={"research_spinoff_from": session_id},
+        ))
+        session_manager.save_sessions()
+
+        return {
+            "session_id": new_sid,
+            "name": new_name,
+            "source_count": len(sources),
+        }
+
+    return router

routes/session_routes.py

@@ -162,7 +162,7 @@ def _persist_session_headers(session_id: str, headers: dict | None) -> None:
         db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
         if db_session:
             db_session.headers = headers or {}
-            db_session.updated_at = utcnow_naive()
+            db_session.updated_at = datetime.utcnow()
             db.commit()
     except Exception:
         db.rollback()
@@ -207,7 +207,6 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
     """Setup session routes with the provided manager and config"""
 
     REQUEST_TIMEOUT = config.get("REQUEST_TIMEOUT", 20)
-    SESSION_MODEL_VALIDATION_TIMEOUT = min(float(REQUEST_TIMEOUT or 20), 3.0)
     OPENAI_API_KEY = config.get("OPENAI_API_KEY")
     SESSIONS_FILE = config.get("SESSIONS_FILE")
     
@@ -224,8 +223,8 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         # purge exists only to catch ghosts the frontend missed (tab close,
         # crash). Only clean up rows old enough to be definitely orphaned.
         try:
-            from datetime import timedelta as _td
+            from datetime import datetime as _dt, timedelta as _td
-            _cutoff = utcnow_naive() - _td(minutes=10)
+            _cutoff = _dt.utcnow() - _td(minutes=10)
             _purge_db = SessionLocal()
             try:
                 from core.database import ChatMessage as _DbMsg
@@ -375,7 +374,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             from src.llm_core import list_model_ids
             ids = list_model_ids(
                 endpoint_url,
-                timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
+                timeout=REQUEST_TIMEOUT,
                 headers=validation_headers,
                 owner=user,
                 endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -395,7 +394,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             req_base = _os.path.basename(model_to_use.rstrip("/"))
             avail = list_model_ids(
                 endpoint_url,
-                timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
+                timeout=REQUEST_TIMEOUT,
                 headers=validation_headers,
                 owner=user,
                 endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -471,7 +470,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == sid).first()
                 if db_session:
                     db_session.folder = folder if folder else None
-                    db_session.updated_at = utcnow_naive()
+                    db_session.updated_at = datetime.utcnow()
                     db.commit()
                     result["folder"] = folder if folder else None
             finally:
@@ -518,7 +517,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                     db_session.model = model
                     db_session.endpoint_url = endpoint_url
                     db_session.headers = session.headers or {}
-                    db_session.updated_at = utcnow_naive()
+                    db_session.updated_at = datetime.utcnow()
                     db.commit()
             finally:
                 db.close()
@@ -647,7 +646,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == sid).first()
                 if db_session:
                     db_session.archived = True
-                    db_session.updated_at = utcnow_naive()
+                    db_session.updated_at = datetime.utcnow()
                     db.commit()
                     
                     # Update in memory if it exists
@@ -681,7 +680,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             if not db_session:
                 raise HTTPException(404, f"Session {sid} not found")
             db_session.archived = False
-            db_session.updated_at = utcnow_naive()
+            db_session.updated_at = datetime.utcnow()
             db.commit()
             # Reload into session manager so it appears in the active list
             try:
@@ -891,7 +890,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
                 if db_session:
                     db_session.is_important = important
-                    db_session.updated_at = utcnow_naive()
+                    db_session.updated_at = datetime.utcnow()
                     db.commit()
 
                     # Update in memory if it exists
@@ -980,7 +979,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
             metadata={
                 "compacted": True,
                 "summarized_count": len(older),
-                "timestamp": utcnow_naive().isoformat(),
+                "timestamp": datetime.utcnow().isoformat(),
             },
         )
         new_history = [summary_msg] + recent
@@ -1257,7 +1256,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                 db_session = db_session_q.first()
                 if db_session:
                     db_session.folder = folder_name
-                    db_session.updated_at = utcnow_naive()
+                    db_session.updated_at = datetime.utcnow()
                     updated += 1
             db.commit()
         except Exception as e:

routes/shell_routes.py

@@ -16,11 +16,6 @@ from pathlib import Path
 from typing import Dict, Any
 from core.platform_compat import IS_APPLE_SILICON, which_tool
 from core.middleware import INTERNAL_TOOL_USER
-from src.host_docker_access import (
-    HOST_DOCKER_ACCESS_HINT,
-    host_docker_access_enabled as _host_docker_access_enabled,
-    running_in_container as _running_in_container,
-)
 from src.optional_deps import prepare_optional_dependency_import
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
@@ -108,17 +103,32 @@ logger = logging.getLogger(__name__)
 PTY_SUPPORTED = pty is not None and fcntl is not None and hasattr(os, "setsid")
 
 
-DOCKER_IN_CONTAINER_HINT = HOST_DOCKER_ACCESS_HINT
+DOCKER_IN_CONTAINER_HINT = (
+    "Not available inside the Odysseus container by design. The image ships no "
+    "docker CLI and no host socket is mounted. Run Docker-backed launches on a "
+    "remote server, where docker is checked over SSH. Mounting /var/run/docker.sock "
+    "into the container would grant it host-root access, so only do that if you "
+    "accept that risk."
+)
+
+
+def _running_in_container(dockerenv_path="/.dockerenv", cgroup_path="/proc/1/cgroup"):
+    if os.path.exists(dockerenv_path):
+        return True
+    try:
+        with open(cgroup_path, "r", encoding="utf-8") as fh:
+            contents = fh.read()
+    except OSError:
+        return False
+    return any(token in contents for token in ("docker", "containerd", "kubepods"))
 
 
 DockerRowStatus = namedtuple("DockerRowStatus", ["applicable", "install_hint"])
 PackageUpdateStatus = namedtuple("PackageUpdateStatus", ["available", "note"])
 
 
-def _docker_row_status(
+def _docker_row_status(*, on_remote, in_container, installed, default_hint):
-    *, on_remote, in_container, installed, default_hint, host_docker_access=False
+    local_docker_unavailable = not on_remote and in_container and not installed
-):
-    local_docker_unavailable = not on_remote and in_container and not host_docker_access
     if local_docker_unavailable:
         return DockerRowStatus(applicable=False, install_hint=DOCKER_IN_CONTAINER_HINT)
     return DockerRowStatus(applicable=True, install_hint=default_hint)
@@ -164,8 +174,6 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
         return bool(binaries.get("llama-server") or dists.get("llama-cpp-python"))
     if name == "sglang":
         return bool(dists.get("sglang") or modules.get("sglang", {}).get("real_module"))
-    if name == "mlx_lm":
-        return bool(dists.get("mlx-lm") or modules.get("mlx_lm", {}).get("real_module"))
     if name == "diffusers":
         return bool(
             (dists.get("diffusers") or modules.get("diffusers", {}).get("real_module"))
@@ -212,10 +220,6 @@ def _package_status_note(name: str, probe: dict) -> str:
         if _package_installed_from_probe(name, probe):
             return f"diffusers {dists.get('diffusers', 'available')} with torch {dists.get('torch', 'available')}"
         return "Diffusers serving needs both diffusers and torch."
-    if name == "mlx_lm":
-        if _package_installed_from_probe(name, probe):
-            return f"MLX LM {dists.get('mlx-lm', 'available')}"
-        return "MLX serving needs mlx-lm on an Apple Silicon Mac."
     if name in dists:
         return f"{name} {dists[name]}"
     return ""
@@ -313,14 +317,12 @@ dist_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-cpp-python'],
     'sglang':['sglang'],
-    'mlx_lm':['mlx-lm'],
     'diffusers':['diffusers','torch'],
     'hf_transfer':['hf-transfer','hf_transfer'],
 }}
 bin_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-server'],
-    'tmux':['tmux'],
 }}
 
 def add_user_install_bins_to_path():
@@ -333,8 +335,6 @@ def add_user_install_bins_to_path():
     candidates.append(os.path.expanduser('~/llama.cpp/build/bin'))
     candidates.append(os.path.expanduser('~/llama.cpp/build-vulkan/bin'))
     candidates.append(os.path.expanduser('~/.local/bin'))
-    candidates.append('/opt/homebrew/bin')
-    candidates.append('/usr/local/bin')
     parts = os.environ.get('PATH', '').split(os.pathsep) if os.environ.get('PATH') else []
     changed = False
     for path in reversed([p for p in candidates if p]):
@@ -409,47 +409,6 @@ class ShellExecRequest(BaseModel):
     use_tmux: bool = False  # run in tmux session (survives browser disconnect)
 
 
-_REMOTE_TMUX_PATH_PREFIX = 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; '
-
-
-def _normalize_legacy_remote_tmux_exec(command: str) -> str:
-    """Repair stale frontend Cookbook tmux SSH commands.
-
-    Older loaded JS sends `ssh host 'tmux capture-pane ...'`. On macOS/Homebrew
-    remotes, non-login SSH shells often lack /opt/homebrew/bin, so tmux is
-    installed but the capture/kill command returns nothing. Keep this narrowly
-    scoped to SSH commands whose remote shell starts with `tmux `.
-    """
-    cmd = command or ""
-    if _REMOTE_TMUX_PATH_PREFIX in cmd or not cmd.lstrip().startswith("ssh "):
-        return cmd
-    try:
-        parts = shlex.split(cmd)
-    except Exception:
-        return cmd
-    if not parts or parts[0] != "ssh":
-        return cmd
-    remote_idx = -1
-    i = 1
-    while i < len(parts):
-        part = parts[i]
-        if part in {"-p", "-o", "-i", "-F", "-J", "-l", "-S", "-W", "-b", "-c", "-m"}:
-            i += 2
-            continue
-        if part.startswith("-"):
-            i += 1
-            continue
-        remote_idx = i
-        break
-    if remote_idx < 0 or remote_idx + 1 >= len(parts):
-        return cmd
-    remote_cmd = " ".join(parts[remote_idx + 1:]).strip()
-    if not remote_cmd.startswith("tmux "):
-        return cmd
-    repaired = parts[:remote_idx + 1] + [_REMOTE_TMUX_PATH_PREFIX + remote_cmd]
-    return shlex.join(repaired)
-
-
 async def _create_shell(command: str, **kwargs):
     """Spawn a shell subprocess for `command`.
 
@@ -866,10 +825,6 @@ def setup_shell_routes() -> APIRouter:
         if not cmd:
             return {"stdout": "", "stderr": "No command provided", "exit_code": 1}
 
-        fixed_cmd = _normalize_legacy_remote_tmux_exec(cmd)
-        if fixed_cmd != cmd:
-            logger.info("Rewrote legacy remote tmux exec command with Homebrew PATH")
-            cmd = fixed_cmd
         logger.info("User shell exec requested: length=%d", len(cmd))
         result = await _exec_shell(
             cmd, timeout=req.timeout if req.timeout is not None else EXEC_TIMEOUT
@@ -1189,13 +1144,6 @@ def setup_shell_routes() -> APIRouter:
                 "category": "LLM",
                 "target": "remote",
             },
-            {
-                "name": "mlx_lm",
-                "pip": "mlx-lm",
-                "desc": "Serve MLX-format models on Apple Silicon Macs",
-                "category": "LLM",
-                "target": "remote",
-            },
             {
                 "name": "APFEL",
                 "pip": "",
@@ -1211,7 +1159,7 @@ def setup_shell_routes() -> APIRouter:
             {
                 "name": "diffusers",
                 "pip": "diffusers[torch]",
-                "desc": "Image generation/editing pipelines (SD, Flux) with PyTorch",
+                "desc": "Image generation pipelines (SD, Flux) with PyTorch",
                 "category": "Image",
                 "target": "remote",
             },
@@ -1341,9 +1289,9 @@ def setup_shell_routes() -> APIRouter:
                 for name in all_system_names:
                     qn = shlex.quote(name)
                     checks.append(
-                        f"PATH=\"$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH\"; if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
+                        f"if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
                     )
-                checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || { [ \"$(uname -s 2>/dev/null)\" = \"Darwin\" ] && echo ID=macos; } || true")
+                checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || true")
                 inner = " ; ".join(checks)
                 argv = _ssh_base_argv(host, ssh_port) + [inner]
                 proc = await asyncio.create_subprocess_exec(
@@ -1562,9 +1510,6 @@ def setup_shell_routes() -> APIRouter:
                     in_container=_running_in_container() if not on_remote else False,
                     installed=pkg["installed"],
                     default_hint=pkg.get("install_hint"),
-                    host_docker_access=(
-                        _host_docker_access_enabled() if not on_remote else False
-                    ),
                 )
                 pkg["applicable"] = status.applicable
                 pkg["install_hint"] = status.install_hint
@@ -1600,7 +1545,6 @@ def setup_shell_routes() -> APIRouter:
             "onnxruntime",
             "hdbscan",
             "vllm",
-            "mlx-lm",
         }
         if pip_name not in known:
             return {"ok": False, "error": f"Unknown package: {pip_name}"}
@@ -1647,19 +1591,6 @@ def setup_shell_routes() -> APIRouter:
                 elif n == "g++": out += ["gcc-c++"]
                 else: out.append(n)
             return out
-        def _apk(names):
-            out = []
-            for n in names:
-                if n == "build-essential": out.append("build-base")
-                else: out.append(n)
-            return out
-        def _zypper(names):
-            out = []
-            for n in names:
-                if n == "build-essential": out += ["gcc-c++", "make"]
-                elif n == "g++": out.append("gcc-c++")
-                else: out.append(n)
-            return out
         def _brew(names):
             return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
         # Build a single shell snippet that detects the package manager and
@@ -1668,8 +1599,6 @@ def setup_shell_routes() -> APIRouter:
         apt_pkgs = " ".join(shlex.quote(p) for p in _apt(pkgs))
         pac_pkgs = " ".join(shlex.quote(p) for p in _pacman(pkgs))
         dnf_pkgs = " ".join(shlex.quote(p) for p in _dnf(pkgs))
-        apk_pkgs = " ".join(shlex.quote(p) for p in _apk(pkgs))
-        zypper_pkgs = " ".join(shlex.quote(p) for p in _zypper(pkgs))
         brew_pkgs = " ".join(shlex.quote(p) for p in _brew(pkgs))
         # Error messages go to stderr (>&2) so the route's error field
         # gets populated. Without the redirect, `echo "ERROR…"` on stdout
@@ -1677,28 +1606,18 @@ def setup_shell_routes() -> APIRouter:
         # bare "HTTP 200" instead of surfacing the real reason.
         script = (
             'set -e; '
-            'BREW="$(command -v brew 2>/dev/null || true)"; '
+            'if ! sudo -n true 2>/dev/null; then '
-            'if [ -z "$BREW" ] && [ -x /opt/homebrew/bin/brew ]; then BREW=/opt/homebrew/bin/brew; fi; '
+            '  echo "ERROR: passwordless sudo unavailable on this target. Run once: sudo apt install -y ' + " ".join(pkgs) + ' (or your distro equivalent: pacman -S, dnf install, brew install). After that, Cookbook can install the rest." >&2; exit 2; fi; '
-            'if [ -z "$BREW" ] && [ -x /usr/local/bin/brew ]; then BREW=/usr/local/bin/brew; fi; '
-            'if [ -n "$BREW" ]; then '
-            f'  if [ -z "{brew_pkgs}" ]; then echo "Nothing to install with brew for requested packages." >&2; exit 4; fi; "$BREW" install {brew_pkgs}; exit $?; '
-            'fi; '
-            'if [ "$(id -u)" = "0" ]; then SUDO=""; '
-            'elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then SUDO="sudo -n"; '
-            'else '
-            '  echo "ERROR: this target needs sudo for its OS package manager, but passwordless sudo is unavailable. Open a terminal on the target and run the shown install command once, then retry in Cookbook." >&2; exit 2; fi; '
             'if command -v apt-get >/dev/null 2>&1; then '
-            f'  $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq && $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
+            f'  sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq && sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
             'elif command -v pacman >/dev/null 2>&1; then '
-            f'  $SUDO pacman -Sy --needed --noconfirm {pac_pkgs}; '
+            f'  sudo -n pacman -Sy --needed --noconfirm {pac_pkgs}; '
             'elif command -v dnf >/dev/null 2>&1; then '
-            f'  $SUDO dnf install -y {dnf_pkgs}; '
+            f'  sudo -n dnf install -y {dnf_pkgs}; '
-            'elif command -v apk >/dev/null 2>&1; then '
+            'elif command -v brew >/dev/null 2>&1; then '
-            f'  $SUDO apk add --no-interactive {apk_pkgs}; '
+            f'  brew install {brew_pkgs}; '
-            'elif command -v zypper >/dev/null 2>&1; then '
-            f'  $SUDO zypper --non-interactive install {zypper_pkgs}; '
             'else '
-            '  echo "ERROR: no supported package manager (apt/pacman/dnf/apk/zypper/brew) on this target." >&2; exit 3; fi'
+            '  echo "ERROR: no supported package manager (apt/pacman/dnf/brew) on this target." >&2; exit 3; fi'
         )
         try:
             if host:

routes/task_routes.py

@@ -11,14 +11,10 @@ from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
 from core.database import SessionLocal, ScheduledTask, TaskRun
+from core.middleware import INTERNAL_TOOL_USER
 from core.constants import internal_api_base
 from src.auth_helpers import get_current_user
 from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
-from src.task_action_policy import (
-    ADMIN_ONLY_TASK_ACTIONS,
-    is_admin_only_task_action,
-    owner_has_admin_task_privileges,
-)
 from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS
 from routes.prefs_routes import _load_for_user, _save_for_user
 
@@ -421,18 +417,28 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 db.close()
         return {"ok": True, "opened": True, "enabled": bool(prefs.get("tasks_enabled")), "resumed": resumed}
 
-    # Actions that execute shell/SSH commands or cross into admin-only
+    # Actions that execute shell/SSH commands — restricted to admins.
-    # Cookbook serving surfaces — restricted to admins.
     # Non-admin users cannot create tasks with these action types via the
     # API. See review CRIT-C.
-    _ADMIN_ONLY_ACTIONS = ADMIN_ONLY_TASK_ACTIONS
+    _ADMIN_ONLY_ACTIONS = {"run_local", "run_script", "ssh_command"}
 
     def _is_admin(user: str | None) -> bool:
-        return owner_has_admin_task_privileges(user)
+        if not user:
-
+            return False
-    def _require_admin_for_task_action(user: str | None, task_type: str | None, action: str | None) -> None:
+        # In-process tool-loopback marker — AuthMiddleware validated
-        if is_admin_only_task_action(task_type, action) and not _is_admin(user):
+        # the internal token + loopback client before stamping this,
-            raise HTTPException(403, f"Action '{action}' requires admin privileges")
+        # so treat as admin-equivalent.
+        if user == INTERNAL_TOOL_USER:
+            return True
+        try:
+            from core.auth import AuthManager
+            auth = AuthManager()
+            if not auth.is_configured:
+                # Unconfigured single-user deploy: trust the local owner.
+                return True
+            return bool(auth.is_admin(user))
+        except Exception:
+            return False
 
     def _validate_then_task_id(db, then_task_id: Optional[str], user: Optional[str], current_task_id: Optional[str] = None) -> Optional[str]:
         target_id = (then_task_id or "").strip()
@@ -460,7 +466,8 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         # Block shell-executing action types for non-admins. action_run_local
         # uses subprocess.run(shell=True) and ssh_command / run_script run
         # arbitrary commands.
-        _require_admin_for_task_action(user, req.task_type, req.action)
+        if req.task_type == "action" and req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
+            raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
         if req.trigger_type == "schedule" and not req.schedule:
             raise HTTPException(400, "Schedule is required for schedule-triggered tasks")
         if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression:
@@ -587,7 +594,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         cache_tables = {
             "summarize_emails": ("email_summaries",),
             "draft_email_replies": ("email_ai_replies",),
-            "email_auto_translate": ("email_translations",),
             "extract_email_events": ("email_calendar_extractions",),
             "learn_sender_signatures": ("sender_signatures",),
             "check_email_urgency": ("email_tags", "email_urgency_alerts"),
@@ -674,10 +680,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
             if user and task.owner != user:
                 raise HTTPException(403, "Access denied")
 
-            next_task_type = req.task_type if req.task_type is not None else task.task_type
-            next_action = req.action if req.action is not None else task.action
-            _require_admin_for_task_action(user, next_task_type, next_action)
-
             if req.name is not None:
                 task.name = req.name
             if req.prompt is not None:
@@ -685,6 +687,9 @@ def setup_task_routes(task_scheduler) -> APIRouter:
             if req.task_type is not None:
                 task.task_type = req.task_type
             if req.action is not None:
+                # Same admin-only gate as create — see CRIT-C.
+                if req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
+                    raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
                 task.action = req.action
             if req.output_target is not None:
                 task.output_target = req.output_target
@@ -801,7 +806,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 raise HTTPException(404, "Task not found")
             if user and task.owner != user:
                 raise HTTPException(403, "Access denied")
-            _require_admin_for_task_action(user, task.task_type, task.action)
             task.status = "active"
             if (task.trigger_type or "schedule") == "schedule":
                 task.next_run = compute_next_run(
@@ -864,7 +868,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 raise HTTPException(404, "Task not found")
             if user and task.owner != user:
                 raise HTTPException(403, "Access denied")
-            _require_admin_for_task_action(user, task.task_type, task.action)
         finally:
             db.close()
         started = await task_scheduler.run_task_now(task_id, force=force)
@@ -890,11 +893,10 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         return {"ok": True, "message": "Task stopped"}
 
     @router.get("/runs/recent")
-    async def list_recent_runs(request: Request, limit: int = 50, max_result_chars: int = 6000):
+    async def list_recent_runs(request: Request, limit: int = 50):
         """Recent task runs across ALL tasks for this owner. Drives the Activity view."""
         user = _owner(request)
         limit = max(1, min(limit, 200))
-        max_result_chars = max(500, min(max_result_chars, 20000))
         db = SessionLocal()
         try:
             q = db.query(TaskRun, ScheduledTask).join(
@@ -928,20 +930,10 @@ def setup_task_routes(task_scheduler) -> APIRouter:
                 deduped.append((r, t))
                 if len(deduped) >= limit:
                     break
-
-            def _clip_run(r: TaskRun) -> dict:
-                d = _run_to_dict(r)
-                for key in ("result", "error"):
-                    val = d.get(key)
-                    if isinstance(val, str) and len(val) > max_result_chars:
-                        d[key] = val[:max_result_chars].rstrip() + "\n\n[Activity preview truncated]"
-                return d
-
             return {
-                "has_more": len(rows) > len(deduped),
                 "runs": [
                     {
-                        **_clip_run(r),
+                        **_run_to_dict(r),
                         "task_name": _display_task_name(t),
                         "task_type": t.task_type or "llm",
                         "action": t.action,
@@ -1054,14 +1046,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
             ).first()
             if not task:
                 raise HTTPException(404, "Not found")
-            if (
-                is_admin_only_task_action(task.task_type, task.action)
-                and not owner_has_admin_task_privileges(task.owner)
-            ):
-                task.status = "paused"
-                task.next_run = None
-                db.commit()
-                raise HTTPException(403, f"Action '{task.action}' requires admin privileges")
         finally:
             db.close()
         started = await task_scheduler.run_task_now(task_id)

routes/upload_routes.py

@@ -6,11 +6,11 @@ import asyncio
 import shutil
 import uuid
 from pathlib import Path
-from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
+from fastapi import APIRouter, Request, File, UploadFile, HTTPException
-from typing import List, Optional
+from typing import List
 import logging
 from core.middleware import require_admin
-from core.database import SessionLocal, GalleryImage, Session as DbSession
+from core.database import SessionLocal, GalleryImage
 from src.auth_helpers import effective_user
 from src.constants import GENERATED_IMAGES_DIR
 from src.upload_handler import count_recent_uploads
@@ -56,17 +56,7 @@ def setup_upload_routes(upload_handler):
 
         raise HTTPException(404, "File not found")
 
-    def _valid_session_id_for_owner(db, session_id: str | None, owner: str | None) -> str | None:
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None) -> str | None:
-        if not session_id:
-            return None
-        sess = db.query(DbSession).filter(DbSession.id == session_id).first()
-        if not sess:
-            return None
-        if owner and sess.owner and sess.owner != owner:
-            return None
-        return session_id
-
-    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
         """Make chat-uploaded images visible in Gallery without changing chat storage."""
         is_image_file = getattr(upload_handler, "is_image_file", None)
         if not callable(is_image_file):
@@ -115,7 +105,6 @@ def setup_upload_routes(upload_handler):
                 prompt=meta.get("name") or "Chat upload",
                 model="chat-upload",
                 owner=owner,
-                session_id=_valid_session_id_for_owner(db, session_id, owner),
                 file_hash=file_hash,
                 width=meta.get("width"),
                 height=meta.get("height"),
@@ -131,14 +120,8 @@ def setup_upload_routes(upload_handler):
             db.close()
     
     @router.post("")
-    async def api_upload(
+    async def api_upload(request: Request, files: List[UploadFile] = File(...)):
-        request: Request,
-        files: List[UploadFile] = File(...),
-        session_id: Optional[str] = Form(None),
-    ):
         """Upload files with enhanced security and organization."""
-        if not isinstance(session_id, str):
-            session_id = None
         if not files:
             raise HTTPException(400, "No files uploaded")
             
@@ -165,7 +148,7 @@ def setup_upload_routes(upload_handler):
             try:
                 owner = effective_user(request)
                 meta = upload_handler.save_upload(u, client_ip, owner=owner)
-                gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
+                gallery_id = _promote_chat_image_to_gallery(meta, owner)
                 item = {
                     "id": meta["id"],
                     "name": meta["name"],
@@ -280,32 +263,6 @@ def setup_upload_routes(upload_handler):
         os.makedirs(cache_dir, exist_ok=True)
         return os.path.join(cache_dir, file_id + ".txt")
 
-    def _sync_gallery_caption_for_upload(info: dict | None, owner: str | None, text: str) -> None:
-        """Copy upload OCR/vision text onto the promoted gallery image row."""
-        if not info:
-            return
-        file_hash = info.get("hash")
-        if not file_hash:
-            return
-        db = SessionLocal()
-        try:
-            q = db.query(GalleryImage).filter(
-                GalleryImage.file_hash == file_hash,
-                GalleryImage.is_active == True,  # noqa: E712
-            )
-            if owner:
-                q = q.filter(GalleryImage.owner == owner)
-            img = q.first()
-            if not img:
-                return
-            img.caption = (text or "").strip()
-            db.commit()
-        except Exception as e:
-            db.rollback()
-            logger.warning("Failed to sync OCR caption to gallery image: %s", e)
-        finally:
-            db.close()
-
     @router.get("/{file_id}/vision")
     async def get_vision_text(request: Request, file_id: str, force: int = 0):
         """Return the vision-model OCR/description for an uploaded image.
@@ -332,9 +289,7 @@ def setup_upload_routes(upload_handler):
         if not force and os.path.exists(cache_path):
             try:
                 with open(cache_path, encoding="utf-8") as f:
-                    cached_text = f.read()
+                    return {"text": f.read(), "cached": True}
-                _sync_gallery_caption_for_upload(info, file_owner or current_user, cached_text)
-                return {"text": cached_text, "cached": True}
             except Exception as e:
                 logger.warning(f"Vision cache read failed for {file_id}: {e}")
         from src.document_processor import analyze_image_with_vl
@@ -348,7 +303,6 @@ def setup_upload_routes(upload_handler):
                 f.write(text)
         except Exception as e:
             logger.warning(f"Vision cache write failed for {file_id}: {e}")
-        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"text": text, "cached": False}
 
     @router.put("/{file_id}/vision")
@@ -379,7 +333,6 @@ def setup_upload_routes(upload_handler):
             raise HTTPException(400, "text must be a string")
         with open(_vision_cache_path(file_id), "w", encoding="utf-8") as f:
             f.write(text)
-        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"ok": True}
 
     async def periodic_rate_limit_cleanup():

scripts/odysseus-gallery

@@ -38,27 +38,23 @@ def _preview_text(value, limit: int = 200) -> str:
     return text[:limit]
 
 
-def _text_field(value) -> str:
-    return value if isinstance(value, str) else ""
-
-
 def _serialize_image(i: "GalleryImage") -> dict:
     return {
         "id": i.id,
-        "filename": _text_field(i.filename),
+        "filename": i.filename,
         "prompt": _preview_text(i.prompt),
-        "model": _text_field(i.model),
+        "model": i.model or "",
-        "size": _text_field(i.size),
+        "size": i.size or "",
-        "tags": _text_field(i.tags),
+        "tags": i.tags or "",
         "favorite": bool(i.favorite),
-        "album_id": _text_field(i.album_id),
+        "album_id": i.album_id or "",
-        "session_id": _text_field(i.session_id),
+        "session_id": i.session_id or "",
         "width": i.width,
         "height": i.height,
         "file_size": i.file_size,
         "taken_at": i.taken_at.isoformat() if i.taken_at else "",
-        "camera_make": _text_field(i.camera_make),
+        "camera_make": i.camera_make or "",
-        "camera_model": _text_field(i.camera_model),
+        "camera_model": i.camera_model or "",
         "created_at": i.created_at.isoformat() if i.created_at else "",
     }
 
@@ -97,11 +93,11 @@ def cmd_show(args):
         if not i:
             fail(f"no image with id {args.id!r}")
         out = _serialize_image(i)
-        out["prompt_full"] = _text_field(i.prompt)
+        out["prompt_full"] = i.prompt or ""
-        out["ai_tags"] = _text_field(i.ai_tags)
+        out["ai_tags"] = i.ai_tags or ""
         out["gps_lat"] = i.gps_lat or ""
         out["gps_lng"] = i.gps_lng or ""
-        out["file_hash"] = _text_field(i.file_hash)
+        out["file_hash"] = i.file_hash or ""
         emit(out, args)
     finally:
         db.close()

scripts/odysseus-mail

@@ -108,8 +108,6 @@ def _q(name: str) -> str:
 
 
 def _split_recipients(value: str) -> list[str]:
-    if not isinstance(value, str):
-        return []
     return [r.strip() for r in (value or "").split(",") if r.strip()]
 
 

scripts/odysseus-notes

@@ -36,9 +36,7 @@ def _load_items(raw) -> list:
         items = json.loads(raw)
     except (TypeError, json.JSONDecodeError):
         return []
-    if not isinstance(items, list):
+    return items if isinstance(items, list) else []
-        return []
-    return [item for item in items if isinstance(item, dict)]
 
 
 def _serialize(n: "Note") -> dict:

services/hwfit/data/hf_models.json

@@ -5113,8 +5113,8 @@
  {
   "name": "deepseek-ai/DeepSeek-V4-Flash",
   "provider": "deepseek-ai",
-  "parameter_count": "158.1B",
+  "parameter_count": "284B",
-  "parameters_raw": 158069433298,
+  "parameters_raw": 284000000000,
   "active_parameters": 13000000000,
   "is_moe": true,
   "min_ram_gb": 200.0,
@@ -5130,40 +5130,15 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 1882337,
+  "hf_downloads": 3542202,
-  "hf_likes": 1651,
+  "hf_likes": 0,
-  "release_date": "2026-06-22"
+  "release_date": "2026-05-15"
- },
- {
-  "name": "deepseek-ai/DeepSeek-V4-Flash-DSpark",
-  "provider": "deepseek-ai",
-  "parameter_count": "165.3B",
-  "parameters_raw": 165265454782,
-  "active_parameters": 13000000000,
-  "is_moe": true,
-  "active_experts": 6,
-  "min_ram_gb": 170.0,
-  "recommended_ram_gb": 250.0,
-  "min_vram_gb": 165.0,
-  "quantization": "FP8-Mixed",
-  "context_length": 1000000,
-  "use_case": "General-purpose reasoning, long-context",
-  "capabilities": [
-   "long_context",
-   "reasoning",
-   "moe"
-  ],
-  "pipeline_tag": "text-generation",
-  "architecture": "deepseek_v4_moe",
-  "hf_downloads": 4446,
-  "hf_likes": 107,
-  "release_date": "2026-06-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Flash-Base",
   "provider": "deepseek-ai",
-  "parameter_count": "292.0B",
+  "parameter_count": "284B",
-  "parameters_raw": 292021347282,
+  "parameters_raw": 284000000000,
   "active_parameters": 13000000000,
   "is_moe": true,
   "min_ram_gb": 290.0,
@@ -5178,15 +5153,15 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 76030,
+  "hf_downloads": 0,
-  "hf_likes": 256,
+  "hf_likes": 0,
-  "release_date": "2026-04-27"
+  "release_date": "2026-05-15"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Pro",
   "provider": "deepseek-ai",
-  "parameter_count": "861.6B",
+  "parameter_count": "1.6T",
-  "parameters_raw": 861608274846,
+  "parameters_raw": 1600000000000,
   "active_parameters": 49000000000,
   "is_moe": true,
   "min_ram_gb": 1100.0,
@@ -5202,40 +5177,15 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 1154610,
+  "hf_downloads": 0,
-  "hf_likes": 5118,
+  "hf_likes": 0,
-  "release_date": "2026-06-22"
+  "release_date": "2026-05-15"
- },
- {
-  "name": "deepseek-ai/DeepSeek-V4-Pro-DSpark",
-  "provider": "deepseek-ai",
-  "parameter_count": "889.5B",
-  "parameters_raw": 889484881098,
-  "active_parameters": 49000000000,
-  "is_moe": true,
-  "active_experts": 6,
-  "min_ram_gb": 900.0,
-  "recommended_ram_gb": 1250.0,
-  "min_vram_gb": 890.0,
-  "quantization": "FP8-Mixed",
-  "context_length": 1000000,
-  "use_case": "Flagship reasoning, long-context",
-  "capabilities": [
-   "long_context",
-   "reasoning",
-   "moe"
-  ],
-  "pipeline_tag": "text-generation",
-  "architecture": "deepseek_v4_moe",
-  "hf_downloads": 6939,
-  "hf_likes": 241,
-  "release_date": "2026-06-27"
  },
  {
   "name": "deepseek-ai/DeepSeek-V4-Pro-Base",
   "provider": "deepseek-ai",
   "parameter_count": "1.6T",
-  "parameters_raw": 1600790440862,
+  "parameters_raw": 1600000000000,
   "active_parameters": 49000000000,
   "is_moe": true,
   "min_ram_gb": 1700.0,
@@ -5250,9 +5200,9 @@
   ],
   "pipeline_tag": "text-generation",
   "architecture": "deepseek_v4_moe",
-  "hf_downloads": 25387,
+  "hf_downloads": 0,
-  "hf_likes": 305,
+  "hf_likes": 0,
-  "release_date": "2026-04-27"
+  "release_date": "2026-05-15"
  },
  {
   "name": "deepseek-ai/deepseek-coder-6.7b-base",
@@ -13358,106 +13308,6 @@
   "_discovered": true,
   "gguf_sources": []
  },
- {
-  "name": "zai-org/GLM-5.2",
-  "provider": "zai-org",
-  "parameter_count": "753.3B",
-  "parameters_raw": 753329940480,
-  "min_ram_gb": 1510.0,
-  "recommended_ram_gb": 1800.0,
-  "min_vram_gb": 1510.0,
-  "quantization": "BF16",
-  "context_length": 1048576,
-  "use_case": "General purpose reasoning, coding, long-context",
-  "capabilities": [
-   "long_context",
-   "reasoning",
-   "coding",
-   "moe"
-  ],
-  "pipeline_tag": "text-generation",
-  "architecture": "glm_moe_dsa",
-  "hf_downloads": 142547,
-  "hf_likes": 2996,
-  "release_date": "2026-06-23",
-  "is_moe": true,
-  "active_experts": 8,
-  "gguf_sources": [
-   {
-    "repo": "unsloth/GLM-5.2-GGUF",
-    "provider": "unsloth",
-    "file": "UD-Q4_K_M/*.gguf",
-    "quant": "Q4_K_M"
-   }
-  ]
- },
- {
-  "name": "zai-org/GLM-5.2-FP8",
-  "provider": "zai-org",
-  "parameter_count": "753.4B",
-  "parameters_raw": 753375793584,
-  "min_ram_gb": 760.0,
-  "recommended_ram_gb": 900.0,
-  "min_vram_gb": 760.0,
-  "quantization": "FP8",
-  "context_length": 1048576,
-  "use_case": "General purpose reasoning, coding, long-context",
-  "capabilities": [
-   "long_context",
-   "reasoning",
-   "coding",
-   "moe"
-  ],
-  "pipeline_tag": "text-generation",
-  "architecture": "glm_moe_dsa",
-  "hf_downloads": 884226,
-  "hf_likes": 182,
-  "release_date": "2026-06-23",
-  "is_moe": true,
-  "active_experts": 8,
-  "gguf_sources": [
-   {
-    "repo": "unsloth/GLM-5.2-GGUF",
-    "provider": "unsloth",
-    "file": "UD-Q4_K_M/*.gguf",
-    "quant": "Q4_K_M"
-   }
-  ]
- },
- {
-  "name": "unsloth/GLM-5.2-GGUF",
-  "provider": "unsloth",
-  "parameter_count": "753.9B",
-  "parameters_raw": 753864139008,
-  "min_ram_gb": 452.0,
-  "recommended_ram_gb": 620.0,
-  "min_vram_gb": 452.0,
-  "quantization": "Q4_K_M",
-  "context_length": 1048576,
-  "use_case": "General purpose reasoning, coding, long-context (GGUF)",
-  "capabilities": [
-   "long_context",
-   "reasoning",
-   "coding",
-   "moe"
-  ],
-  "pipeline_tag": "text-generation",
-  "architecture": "glm-dsa",
-  "hf_downloads": 180394,
-  "hf_likes": 474,
-  "release_date": "2026-06-23",
-  "is_moe": true,
-  "active_experts": 8,
-  "is_gguf": true,
-  "gguf_sources": [
-   {
-    "repo": "unsloth/GLM-5.2-GGUF",
-    "provider": "unsloth",
-    "file": "UD-Q4_K_M/*.gguf",
-    "quant": "Q4_K_M"
-   }
-  ]
- },
  {
   "name": "cyankiwi/Qwen3.5-35B-A3B-AWQ-4bit",
   "provider": "cyankiwi",
@@ -19105,54 +18955,6 @@
   "active_experts": 8,
   "active_parameters": 13600000000
  },
- {
-  "name": "MiniMaxAI/MiniMax-M3",
-  "provider": "MiniMaxAI",
-  "parameter_count": "427.0B",
-  "parameters_raw": 427040140160,
-  "min_ram_gb": 855.0,
-  "recommended_ram_gb": 1025.0,
-  "min_vram_gb": 855.0,
-  "quantization": "BF16",
-  "context_length": 1000000,
-  "use_case": "Vision, chat, coding, agentic tool use",
-  "capabilities": [
-   "vision",
-   "tool_use",
-   "coding",
-   "moe"
-  ],
-  "pipeline_tag": "image-text-to-text",
-  "architecture": "minimax_m3_vl",
-  "hf_downloads": 192311,
-  "hf_likes": 1267,
-  "release_date": "2026-06-23",
-  "is_moe": true
- },
- {
-  "name": "MiniMaxAI/MiniMax-M3-MXFP8",
-  "provider": "MiniMaxAI",
-  "parameter_count": "440.3B",
-  "parameters_raw": 440279845760,
-  "min_ram_gb": 445.0,
-  "recommended_ram_gb": 560.0,
-  "min_vram_gb": 445.0,
-  "quantization": "MXFP8",
-  "context_length": 1000000,
-  "use_case": "Vision, chat, coding, agentic tool use",
-  "capabilities": [
-   "vision",
-   "tool_use",
-   "coding",
-   "moe"
-  ],
-  "pipeline_tag": "image-text-to-text",
-  "architecture": "minimax_m3_vl",
-  "hf_downloads": 572278,
-  "hf_likes": 43,
-  "release_date": "2026-06-15",
-  "is_moe": true
- },
  {
   "name": "bullerwins/MiniMax-M2.7-REAP-172B-fp8",
   "provider": "bullerwins",

services/hwfit/fit.py

@@ -168,19 +168,6 @@ def _canonical_cpu_backend(system):
     return "cpu_x86"
 
 
-def _is_mlx_model(model, native_q=None):
-    name = (model.get("name") or "").lower()
-    provider = (model.get("provider") or "").lower()
-    fmt = (model.get("format") or "").lower()
-    q = (native_q if native_q is not None else _native_quant(model)).lower()
-    return (
-        q.startswith("mlx-")
-        or provider == "mlx-community"
-        or fmt == "mlx"
-        or name.startswith("mlx-community/")
-    )
-
-
 def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
     """Estimate tok/s. Uses active params for MoE (only active experts run per token).
 
@@ -326,22 +313,6 @@ def _fit_score(required, available):
     return 50
 
 
-def _is_unified_memory_system(system):
-    backend = (system.get("backend") or "").lower()
-    return bool(system.get("unified_memory")) or backend in ("metal", "mps", "apple")
-
-
-def _fit_level_for_budget(required_gb, budget_gb):
-    if not required_gb or not budget_gb or required_gb > budget_gb:
-        return "too_tight"
-    ratio = required_gb / budget_gb
-    if ratio <= 0.50:
-        return "perfect"
-    if ratio <= 0.78:
-        return "good"
-    return "marginal"
-
-
 def _context_score(ctx, use_case):
     target = CONTEXT_TARGET.get(use_case, 4096)
     if ctx >= target:
@@ -545,42 +516,21 @@ def analyze_model(model, system, target_quant=None, scoring_use_case=None, targe
     run_mode, quant, fit_ctx, required_gb = result
 
     # Determine fit level
-    unified_memory = _is_unified_memory_system(system)
+    budget = effective_vram if run_mode == "gpu" else available_ram
-    total_ram = system.get("total_ram_gb") or available_ram
-    unified_budget = max(total_ram or 0, available_ram or 0, effective_vram or 0)
-    budget = unified_budget if unified_memory else (effective_vram if run_mode == "gpu" else available_ram)
     if required_gb > budget:
         return None
     if run_mode == "gpu":
-        if unified_memory:
+        rec = model.get("recommended_ram_gb") or required_gb
-            fit_level = _fit_level_for_budget(required_gb, budget)
+        if rec <= gpu_vram:
-        else:
+            fit_level = "perfect"
-            # GPU-only fit must leave real allocator/KV/runtime headroom. The
+        elif gpu_vram >= required_gb * 1.2:
-            # old check used recommended_ram_gb (or required_gb as a fallback),
-            # which made any model that barely fit VRAM read as "perfect".
-            # On CUDA/vLLM/SGLang that is misleading: 141 GB on a 160 GB box is
-            # runnable, but not a comfortable perfect fit.
-            if gpu_vram >= required_gb * 1.50:
-                fit_level = "perfect"
-            elif gpu_vram >= required_gb * 1.2:
-                fit_level = "good"
-            else:
-                fit_level = "marginal"
-    elif run_mode == "cpu_offload":
-        fit_level = _fit_level_for_budget(required_gb, budget)
-        if fit_level == "perfect":
             fit_level = "good"
-    else:
+        else:
-        fit_level = _fit_level_for_budget(required_gb, budget)
-        if fit_level == "too_tight":
             fit_level = "marginal"
-
+    elif run_mode == "cpu_offload":
-    # Rows that comfortably fit in a huge RAM/unified-memory pool should not all
+        fit_level = "good" if available_ram >= required_gb * 1.2 else "marginal"
-    # look "marginal"; that made 1B-70B CPU/Ollama rows orange on 256 GB systems.
+    else:
-    if fit_level == "marginal" and budget and required_gb <= budget * 0.78:
+        fit_level = "marginal"
-        fit_level = "good"
-    if fit_level == "good" and budget and required_gb <= budget * 0.50 and run_mode != "cpu_offload":
-        fit_level = "perfect"
 
     # Fraction of the model that spills to CPU RAM (drives the offload speed
     # model). When offloading, anything beyond the GPU's VRAM lives in system RAM.
@@ -671,40 +621,6 @@ SORT_KEYS = {
 }
 
 
-def _search_blob(*parts):
-    text = " ".join(str(p or "") for p in parts).lower()
-    compact = re.sub(r"[^a-z0-9]+", "", text)
-    spaced = re.sub(r"[^a-z0-9]+", " ", text).strip()
-    return f"{text} {spaced} {compact}"
-
-
-def _matches_search(model, search):
-    terms = [t for t in re.split(r"\s+", (search or "").strip().lower()) if t]
-    if not terms:
-        return True
-    blob = _search_blob(
-        model.get("name"),
-        model.get("provider"),
-        model.get("architecture"),
-        model.get("quantization"),
-        model.get("format"),
-        model.get("parameter_count"),
-    )
-    for term in terms:
-        norm = re.sub(r"[^a-z0-9]+", "", term)
-        if term not in blob and (not norm or norm not in blob):
-            if re.fullmatch(r"\d+(?:\.\d+)?b?", term):
-                try:
-                    wanted = float(term.rstrip("b"))
-                    actual = params_b(model)
-                except (TypeError, ValueError):
-                    actual = 0
-                if wanted > 0 and actual > 0 and abs(actual - wanted) <= max(5.0, wanted * 0.08):
-                    continue
-            return False
-    return True
-
-
 def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None, target_context=None, fit_only=False):
     """Rank all models against detected hardware. Returns sorted list of fit results.
 
@@ -777,11 +693,10 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
 
     for m in models:
         native_q = _native_quant(m)
-        is_mlx = _is_mlx_model(m, native_q)
 
-        # MLX is Apple Silicon-only. It should never appear on CUDA/ROCm/CPU,
+        # MLX needs the mlx_lm runtime, which Odysseus does not generate serve
-        # but it is first-class on Metal where mlx_lm.server can serve it.
+        # commands for. Hide it on every backend, including Metal.
-        if is_mlx and not apple_silicon:
+        if native_q.startswith("mlx-") or "mlx" in (m.get("name") or "").lower():
             continue
 
         # ROCm support for vLLM/SGLang quantized safetensors is too brittle to
@@ -808,7 +723,7 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
         # Windows is the same: Odysseus only supports llama.cpp on Windows,
         # which requires GGUF. vLLM/SGLang are explicitly blocked, so AWQ/GPTQ
         # models without a GGUF source are unservable there.
-        if (apple_silicon or consumer_amd or is_windows) and not is_mlx and not (m.get("is_gguf") or m.get("gguf_sources")):
+        if (apple_silicon or consumer_amd or is_windows) and not (m.get("is_gguf") or m.get("gguf_sources")):
             continue
 
         # Format filter: AWQ tab -> only AWQ models, FP4 tab -> FP4-family models, etc.
@@ -826,26 +741,13 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
             if quant in ("INT4", "INT8", "W4A16", "W8A8", "W8A16") and native_q != quant:
                 continue
 
-        if search and not _matches_search(m, search):
+        if search:
-            continue
+            name = m.get("name", "").lower()
+            provider = m.get("provider", "").lower()
+            if search.lower() not in name and search.lower() not in provider:
+                continue
 
-        model_quant = quant
+        result = analyze_model(m, system, target_quant=quant, scoring_use_case=(use_case or "general"), target_context=target_context)
-        # UI "Q4" means the user's looking for a 4-bit fit. On multi-GPU
-        # CUDA/vLLM/SGLang boxes, many practical 4-bit models are native AWQ
-        # safetensors, not GGUF Q4_K_M. If we pass Q4_K_M into a prequantized
-        # AWQ row, analyze_model correctly rejects it as the wrong serving
-        # format, but the result is confusing: highlighting Quant/Q4 hides the
-        # exact AWQ rows the machine is built to run. Treat Q4 as AWQ-4bit for
-        # native AWQ rows only on accelerator servers that can serve them.
-        if (
-            quant == "Q4_K_M"
-            and system.get("gpu_count", 1) >= 2
-            and not (apple_silicon or consumer_amd or is_windows)
-            and native_q == "AWQ-4bit"
-        ):
-            model_quant = native_q
-
-        result = analyze_model(m, system, target_quant=model_quant, scoring_use_case=(use_case or "general"), target_context=target_context)
         if result is None:
             continue
 

services/hwfit/hf_discovery.py

@@ -1,374 +0,0 @@
-import json
-import os
-import re
-import time
-import urllib.parse
-import urllib.request
-from email.utils import parsedate_to_datetime
-from pathlib import Path
-
-from src.constants import DATA_DIR
-
-
-HF_COLLECTIONS_URL = "https://huggingface.co/api/collections"
-HW_FIT_CACHE_DIR = Path(DATA_DIR) / "hwfit"
-MLX_COMMUNITY_CACHE = HW_FIT_CACHE_DIR / "mlx_community_models.json"
-HF_COLLECTION_MODELS_CACHE = HW_FIT_CACHE_DIR / "hf_collection_models.json"
-HF_COLLECTION_TTL_SECONDS = 24 * 3600
-
-
-HF_COLLECTION_SOURCES = (
-    {
-        "key": "mlx_community",
-        "owner": "mlx-community",
-        "provider": "mlx-community",
-        "repo_prefix": "mlx-community/",
-        "mlx_only": True,
-    },
-    {
-        "key": "zai_org",
-        "owner": "zai-org",
-        "provider": "zai-org",
-    },
-    {
-        "key": "deepseek_ai",
-        "owner": "deepseek-ai",
-        "provider": "deepseek-ai",
-    },
-    {
-        "key": "minimax_ai",
-        "owner": "MiniMaxAI",
-        "provider": "MiniMaxAI",
-    },
-    {
-        "key": "qwen",
-        "owner": "Qwen",
-        "provider": "Qwen",
-    },
-    {
-        "key": "stepfun_ai",
-        "owner": "stepfun-ai",
-        "provider": "stepfun-ai",
-    },
-    {
-        "key": "google",
-        "owner": "google",
-        "provider": "google",
-    },
-    {
-        "key": "openai",
-        "owner": "openai",
-        "provider": "openai",
-    },
-    {
-        "key": "mistralai",
-        "owner": "mistralai",
-        "provider": "mistralai",
-    },
-    {
-        "key": "meta_llama",
-        "owner": "meta-llama",
-        "provider": "meta-llama",
-    },
-    {
-        "key": "nousresearch",
-        "owner": "NousResearch",
-        "provider": "NousResearch",
-    },
-    {
-        "key": "moonshotai",
-        "owner": "moonshotai",
-        "provider": "moonshotai",
-    },
-    {
-        "key": "mllama",
-        "owner": "mllama",
-        "provider": "mllama",
-    },
-)
-
-
-def _format_params(raw):
-    try:
-        n = int(raw or 0)
-    except (TypeError, ValueError):
-        n = 0
-    if n <= 0:
-        return "", 0
-    if n >= 1_000_000_000_000:
-        return f"{n / 1_000_000_000_000:.3g}T", n
-    if n >= 1_000_000_000:
-        return f"{n / 1_000_000_000:.4g}B", n
-    if n >= 1_000_000:
-        return f"{n / 1_000_000:.4g}M", n
-    if n >= 1_000:
-        return f"{n / 1_000:.4g}K", n
-    return str(n), n
-
-
-def _parse_params_from_name(repo_id):
-    name = (repo_id or "").rsplit("/", 1)[-1]
-    active = None
-    m_active = re.search(r"[-_][Aa](\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name)
-    if m_active:
-        active = int(float(m_active.group(1)) * 1_000_000_000)
-        name = name[: m_active.start()] + name[m_active.end() :]
-    total = None
-    for m in re.finditer(r"(\d+(?:\.\d+)?)[Bb](?![a-zA-Z])", name):
-        total = int(float(m.group(1)) * 1_000_000_000)
-        break
-    if total is None:
-        for m in re.finditer(r"(\d+(?:\.\d+)?)[Mm](?![a-zA-Z])", name):
-            total = int(float(m.group(1)) * 1_000_000)
-            break
-    return total or 0, active
-
-
-def _infer_quant(repo_id, source):
-    name = (repo_id or "").rsplit("/", 1)[-1].lower()
-    if source.get("mlx_only"):
-        if "8bit" in name or "8-bit" in name:
-            return "mlx-8bit"
-        if "6bit" in name or "6-bit" in name:
-            return "mlx-6bit"
-        if "5bit" in name or "5-bit" in name:
-            return "mlx-5bit"
-        if "3bit" in name or "3-bit" in name:
-            return "mlx-3bit"
-        if re.search(r"(^|[-_/])bf16($|[-_/])", name):
-            return "BF16"
-        return "mlx-4bit"
-    if "awq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
-        return "AWQ-8bit"
-    if "awq" in name or "4bit" in name or "4-bit" in name:
-        return "AWQ-4bit"
-    if "gptq" in name and ("8bit" in name or "8-bit" in name or "int8" in name):
-        return "GPTQ-Int8"
-    if "gptq" in name:
-        return "GPTQ-Int4"
-    if "mxfp4" in name or "nvfp4" in name or re.search(r"(^|[-_/])fp4($|[-_/])", name):
-        return "FP4-MoE-Mixed"
-    if "mxfp8" in name or re.search(r"(^|[-_/])fp8($|[-_/])", name):
-        return "FP8-Mixed"
-    if "gguf" in name or "q4_k" in name or "q4-k" in name:
-        return "Q4_K_M"
-    if re.search(r"(^|[-_/])bf16($|[-_/])", name):
-        return "BF16"
-    return "BF16"
-
-
-def _quant_bytes_per_param(quant):
-    return {
-        "BF16": 2.2,
-        "FP8": 1.15,
-        "FP8-Mixed": 1.15,
-        "FP4-MoE-Mixed": 0.62,
-        "AWQ-4bit": 0.62,
-        "AWQ-8bit": 1.15,
-        "GPTQ-Int4": 0.62,
-        "GPTQ-Int8": 1.15,
-        "Q4_K_M": 0.62,
-        "mlx-8bit": 1.25,
-        "mlx-6bit": 0.95,
-        "mlx-5bit": 0.82,
-        "mlx-4bit": 0.70,
-        "mlx-3bit": 0.55,
-    }.get(quant, 2.2)
-
-
-def _infer_context(repo_id, pipeline_tag):
-    text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
-    if any(k in text for k in ("whisper", "asr", "speech-recognition", "tts", "audio", "image", "video", "diffusion")):
-        return 4096
-    if any(k in text for k in ("glm-5.2", "deepseek-v4", "minimax-m3")):
-        return 1_000_000
-    if any(k in text for k in ("qwen3", "glm", "deepseek", "minimax")):
-        return 32768
-    return 32768
-
-
-def _infer_use_case(repo_id, pipeline_tag):
-    text = f"{repo_id or ''} {pipeline_tag or ''}".lower()
-    if any(k in text for k in ("whisper", "asr", "speech-recognition", "transcrib")):
-        return "stt"
-    if any(k in text for k in ("tts", "text-to-speech", "kokoro", "audio")):
-        return "tts"
-    if any(k in text for k in ("image-text", "vision", "vlm", "vl-", "ocr", "multimodal")):
-        return "multimodal"
-    if any(k in text for k in ("code", "coder")):
-        return "coding"
-    if any(k in text for k in ("reason", "thinking", "thinker", "r1")):
-        return "reasoning"
-    return "general"
-
-
-def _entry_from_collection_item(collection, item, source):
-    repo_id = item.get("id") or ""
-    if item.get("type") != "model" or not repo_id:
-        return None
-    repo_prefix = source.get("repo_prefix")
-    if repo_prefix and not repo_id.startswith(repo_prefix):
-        return None
-    raw_params = item.get("numParameters") or 0
-    active = None
-    if not raw_params:
-        raw_params, active = _parse_params_from_name(repo_id)
-    param_label, raw_params = _format_params(raw_params)
-    if not raw_params:
-        return None
-
-    quant = _infer_quant(repo_id, source)
-    pipeline_tag = item.get("pipeline_tag") or ""
-    min_ram = round((raw_params / 1_000_000_000) * _quant_bytes_per_param(quant) + 0.8, 1)
-    last_modified = item.get("lastModified") or collection.get("lastUpdated") or ""
-    release_date = ""
-    if last_modified:
-        try:
-            release_date = parsedate_to_datetime(last_modified).date().isoformat()
-        except Exception:
-            release_date = str(last_modified)[:10]
-
-    entry = {
-        "name": repo_id,
-        "provider": source.get("provider") or repo_id.split("/", 1)[0],
-        "parameter_count": param_label,
-        "parameters_raw": raw_params,
-        "min_ram_gb": min_ram,
-        "recommended_ram_gb": round(min_ram * 1.3 + 0.5, 1),
-        "min_vram_gb": 0.0 if source.get("mlx_only") else min_ram,
-        "quantization": quant,
-        "context_length": _infer_context(repo_id, pipeline_tag),
-        "use_case": _infer_use_case(repo_id, pipeline_tag),
-        "capabilities": ["mlx"] if source.get("mlx_only") else ["vllm", "sglang"],
-        "pipeline_tag": pipeline_tag,
-        "architecture": "",
-        "hf_downloads": int(item.get("downloads") or 0),
-        "hf_likes": int(item.get("likes") or 0),
-        "release_date": release_date,
-        "format": "mlx" if source.get("mlx_only") else "safetensors",
-        "collection": collection.get("title") or "",
-        "description": collection.get("description") or "",
-        "_discovered": True,
-        "_source": "hf_collections",
-        "_source_owner": source.get("owner") or "",
-    }
-    if source.get("mlx_only"):
-        entry["mlx_only"] = True
-    if quant == "Q4_K_M":
-        entry["is_gguf"] = True
-        entry["format"] = "gguf"
-        entry["capabilities"] = ["llama.cpp"]
-    if active:
-        entry["is_moe"] = True
-        entry["active_parameters"] = active
-    return entry
-
-
-def _next_link(header):
-    if not header:
-        return None
-    m = re.search(r'<([^>]+)>;\s*rel="next"', header)
-    return m.group(1) if m else None
-
-
-def fetch_collection_models(source, timeout=20, max_pages=20):
-    params = urllib.parse.urlencode({
-        "owner": source["owner"],
-        "limit": "100",
-        "expand": "true",
-    })
-    url = f"{HF_COLLECTIONS_URL}?{params}"
-    models = {}
-    pages = 0
-    while url and pages < max_pages:
-        req = urllib.request.Request(url, headers={"User-Agent": "odysseus-hwfit/1.0"})
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            payload = json.load(resp)
-            url = _next_link(resp.headers.get("Link"))
-        pages += 1
-        if not isinstance(payload, list):
-            break
-        for collection in payload:
-            if not isinstance(collection, dict):
-                continue
-            for item in collection.get("items") or []:
-                if not isinstance(item, dict):
-                    continue
-                entry = _entry_from_collection_item(collection, item, source)
-                if entry and entry["name"] not in models:
-                    models[entry["name"]] = entry
-    rows = list(models.values())
-    rows.sort(key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""), reverse=True)
-    return rows
-
-
-def _load_cache(path):
-    try:
-        with path.open(encoding="utf-8") as f:
-            data = json.load(f)
-        rows = data.get("models") if isinstance(data, dict) else data
-        return rows if isinstance(rows, list) else []
-    except (OSError, ValueError):
-        return []
-
-
-def _write_cache(path, source, rows):
-    path.parent.mkdir(parents=True, exist_ok=True)
-    payload = {
-        "source": source,
-        "fetched_at": int(time.time()),
-        "count": len(rows),
-        "models": rows,
-    }
-    tmp = path.with_suffix(".json.tmp")
-    tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
-    os.replace(tmp, path)
-
-
-def load_cached_mlx_community_models():
-    return _load_cache(MLX_COMMUNITY_CACHE)
-
-
-def load_cached_hf_collection_models():
-    return _load_cache(HF_COLLECTION_MODELS_CACHE)
-
-
-def _cache_fresh(path):
-    try:
-        return (time.time() - path.stat().st_mtime) < HF_COLLECTION_TTL_SECONDS
-    except OSError:
-        return False
-
-
-def refresh_mlx_community_cache(force=False):
-    if not force and _cache_fresh(MLX_COMMUNITY_CACHE):
-        return load_cached_mlx_community_models()
-    source = next(s for s in HF_COLLECTION_SOURCES if s["key"] == "mlx_community")
-    rows = fetch_collection_models(source)
-    _write_cache(MLX_COMMUNITY_CACHE, "https://huggingface.co/mlx-community/collections", rows)
-    return rows
-
-
-def refresh_hf_collection_models_cache(force=False):
-    if not force and _cache_fresh(HF_COLLECTION_MODELS_CACHE):
-        return load_cached_hf_collection_models()
-    rows_by_name = {}
-    for source in HF_COLLECTION_SOURCES:
-        if source["key"] == "mlx_community":
-            continue
-        try:
-            for row in fetch_collection_models(source):
-                rows_by_name.setdefault(row["name"], row)
-        except Exception:
-            # Keep partial refreshes useful. A temporary DNS/provider issue for
-            # one brand should not invalidate the other cached collection rows.
-            continue
-    rows = sorted(
-        rows_by_name.values(),
-        key=lambda x: (x.get("hf_downloads") or 0, x.get("release_date") or ""),
-        reverse=True,
-    )
-    if rows:
-        _write_cache(HF_COLLECTION_MODELS_CACHE, "https://huggingface.co/collections", rows)
-        return rows
-    return load_cached_hf_collection_models()

services/hwfit/models.py

@@ -13,7 +13,7 @@ QUANT_BPP = {
     "AWQ-4bit": 0.50, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.50, "GPTQ-Int8": 1.0,
     "QAT-INT4": 0.50, "QAT-INT8": 1.0,
-    "mlx-3bit": 0.42, "mlx-4bit": 0.55, "mlx-5bit": 0.65, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
+    "mlx-4bit": 0.55, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
     # DeepSeek-V4-style mixed: MoE experts in FP4 (bulk), attention + non-
     # expert dense in FP8, embeddings/LM head in BF16. By weight count the
     # experts dominate so the effective BPP sits closer to FP4 than FP8.
@@ -32,7 +32,7 @@ QUANT_SPEED_MULT = {
     "AWQ-4bit": 1.2, "AWQ-8bit": 0.85,
     "GPTQ-Int4": 1.2, "GPTQ-Int8": 0.85,
     "QAT-INT4": 1.15, "QAT-INT8": 0.85,
-    "mlx-3bit": 1.25, "mlx-4bit": 1.15, "mlx-5bit": 1.05, "mlx-6bit": 1.0, "mlx-8bit": 0.85,
+    "mlx-4bit": 1.15, "mlx-8bit": 0.85, "mlx-6bit": 1.0,
     "FP4-MoE-Mixed": 1.10,  # slightly slower than pure FP4 because of mixed-dtype dispatch
     "FP8-Mixed": 0.85,
 }
@@ -53,7 +53,7 @@ QUANT_QUALITY_PENALTY = {
     # QAT-INT4 build lands far closer to bf16 than a post-training Q4/INT4
     # (Google reports near-bf16 quality). Penalize it lightly, not like Q4_K_M.
     "QAT-INT4": -1.0, "QAT-INT8": 0.0,
-    "mlx-3bit": -8.0, "mlx-4bit": -4.0, "mlx-5bit": -2.5, "mlx-6bit": -1.5, "mlx-8bit": -0.5,
+    "mlx-4bit": -4.0, "mlx-8bit": -0.5, "mlx-6bit": -1.5,
     # DeepSeek-V4 mixed: only MoE experts at FP4 (the rest is FP8/BF16),
     # so the realized quality is much closer to FP8 than to pure FP4 —
     # the activation-sensitive layers stay high-precision. ~0 penalty.
@@ -70,7 +70,7 @@ QUANT_BYTES_PER_PARAM = {
     "AWQ-4bit": 0.5, "AWQ-8bit": 1.0,
     "GPTQ-Int4": 0.5, "GPTQ-Int8": 1.0,
     "QAT-INT4": 0.5, "QAT-INT8": 1.0,
-    "mlx-3bit": 0.375, "mlx-4bit": 0.5, "mlx-5bit": 0.625, "mlx-6bit": 0.75, "mlx-8bit": 1.0,
+    "mlx-4bit": 0.5, "mlx-8bit": 1.0, "mlx-6bit": 0.75,
     "FP4-MoE-Mixed": 0.55,
     "FP8-Mixed": 1.0,
 }
@@ -87,11 +87,8 @@ PREQUANTIZED_PREFIXES = (
 
 def infer_quantization_from_name(name):
     n = (name or "").lower()
-    model_name = n.rsplit("/", 1)[-1]
     if "nvfp4" in n:
         return "NVFP4"
-    if re.search(r"(^|[-_/])bf16($|[-_/])", model_name):
-        return "BF16"
     if "mxfp4" in n:
         return "MXFP4"
     if re.search(r"(^|[-_/])nf4($|[-_/])", n):
@@ -109,12 +106,8 @@ def infer_quantization_from_name(name):
         return "AWQ-8bit" if is8 else "AWQ-4bit"
     if "gptq" in n:
         return "GPTQ-Int8" if is8 else "GPTQ-Int4"
-    if n.startswith("mlx-community/") or "mlx" in model_name:
+    if "mlx" in n:
-        if "3bit" in model_name:
+        if "6bit" in n:
-            return "mlx-3bit"
-        if "5bit" in model_name:
-            return "mlx-5bit"
-        if "6bit" in model_name:
             return "mlx-6bit"
         return "mlx-8bit" if is8 else "mlx-4bit"
     if "fp8" in n:
@@ -267,75 +260,15 @@ def infer_use_case(model):
 
 _models_cache = None
 
-def _load_model_file(path):
-    try:
-        with open(path, encoding="utf-8") as f:
-            loaded = json.load(f)
-            return loaded if isinstance(loaded, list) else []
-    except (FileNotFoundError, json.JSONDecodeError):
-        return []
-
-def reset_model_cache():
-    global _models_cache
-    _models_cache = None
-
-def refresh_dynamic_catalogs(force=False):
-    """Refresh API-backed model catalogs and invalidate the merged cache.
-
-    The bundled JSON files remain the offline fallback. Dynamic catalogs live
-    under DATA_DIR so runtime refreshes do not dirty the source tree.
-    """
-    from services.hwfit.hf_discovery import (
-        refresh_hf_collection_models_cache,
-        refresh_mlx_community_cache,
-    )
-
-    refreshed = {
-        "mlx_community": len(refresh_mlx_community_cache(force=force)),
-        "hf_collections": len(refresh_hf_collection_models_cache(force=force)),
-    }
-    reset_model_cache()
-    return refreshed
-
 def get_models():
     global _models_cache
     if _models_cache is None:
         data_path = os.path.join(os.path.dirname(__file__), "data", "hf_models.json")
-        static_mlx_path = os.path.join(os.path.dirname(__file__), "data", "mlx_community_models.json")
         try:
-            from services.hwfit.hf_discovery import (
+            with open(data_path, encoding="utf-8") as f:
-                load_cached_hf_collection_models,
+                _models_cache = [_normalize_model_entry(m) for m in json.load(f)]
-                load_cached_mlx_community_models,
+        except (FileNotFoundError, json.JSONDecodeError):
-            )
+            _models_cache = []
-            dynamic_mlx_models = load_cached_mlx_community_models()
-            dynamic_hf_models = load_cached_hf_collection_models()
-        except Exception:
-            dynamic_mlx_models = []
-            dynamic_hf_models = []
-        seen = set()
-        rows = []
-        def _append_models(models):
-            for model in models:
-                if not isinstance(model, dict):
-                    continue
-                name = model.get("name")
-                if not name or name in seen:
-                    continue
-                seen.add(name)
-                rows.append(_normalize_model_entry(model))
-
-        for model in _load_model_file(data_path):
-            if not isinstance(model, dict):
-                continue
-            name = model.get("name")
-            if not name or name in seen:
-                continue
-            seen.add(name)
-            rows.append(_normalize_model_entry(model))
-        _append_models(dynamic_hf_models)
-        _append_models(dynamic_mlx_models)
-        _append_models(_load_model_file(static_mlx_path))
-        _models_cache = rows
     return _models_cache
 
 

services/hwfit/profiles.py

@@ -103,9 +103,6 @@ def compute_serve_profiles(system, model, serve_weights_gb=None, serve_quant=Non
     in the actual serving knobs (n_cpu_moe, KV-cache type, context). serve_quant
     is the file's quant label (e.g. "Q4_K_M") just for display.
     """
-    if not isinstance(system, dict) or not isinstance(model, dict):
-        return []
-
     vram = float(system.get("gpu_vram_gb") or 0)
     if vram <= 0:
         return []

services/search/content.py

@@ -8,13 +8,11 @@ import os
 import re
 import logging
 import socket
-import ssl
 from datetime import datetime, timedelta
-from typing import Iterable, List, cast
+from typing import List
 from urllib.parse import urljoin, urlparse
 
 import httpx
-import httpcore
 from bs4 import BeautifulSoup
 
 from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
@@ -93,148 +91,6 @@ def _public_http_url(url: str) -> bool:
         return False
 
 
-def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
-    parsed = urlparse(url)
-    if parsed.scheme not in ("http", "https") or not parsed.hostname:
-        raise httpx.RequestError(f"Blocked non-public URL: {url}")
-    host = (parsed.hostname or "").strip().lower()
-    if host in ("localhost", "metadata", "metadata.google.internal"):
-        raise httpx.RequestError(f"Blocked non-public hostname: {host}")
-    try:
-        ip = ipaddress.ip_address(host)
-        if _is_private_address(ip):
-            raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
-        return [ip]
-    except httpx.RequestError:
-        raise
-    except ValueError:
-        pass
-    addrs = _resolve_hostname_ips(host)
-    if not addrs or any(_is_private_address(a) for a in addrs):
-        raise httpx.RequestError(f"Blocked non-public URL: {url}")
-    return addrs
-
-
-class _PinnedBackend(httpcore.NetworkBackend):
-    """Network backend that connects to a pre-resolved IP.
-
-    httpcore derives the TLS SNI and the ``Host`` header from the URL's
-    origin, not from the host argument passed to ``connect_tcp``. So
-    routing the TCP connect to a resolved IP while leaving the URL
-    untouched keeps SNI / vhost behaviour correct and closes the
-    DNS-rebinding TOCTOU between the SSRF check and the connect.
-    """
-
-    def __init__(self, ip: ipaddress._BaseAddress):
-        self._ip = str(ip)
-        self._real = httpcore.SyncBackend()
-
-    def connect_tcp(
-        self,
-        host: str,
-        port: int,
-        timeout: float | None = None,
-        local_address: str | None = None,
-        socket_options=None,
-    ):
-        return self._real.connect_tcp(
-            self._ip, port, timeout, local_address, socket_options
-        )
-
-    def connect_unix_socket(self, path, timeout=None, socket_options=None):
-        return self._real.connect_unix_socket(path, timeout, socket_options)
-
-    def sleep(self, seconds: float) -> None:
-        return self._real.sleep(seconds)
-
-
-# Map httpcore exception classes to their httpx equivalents. Built
-# once at import time from the public exception classes; avoids any
-# import of httpx's private transport machinery. httpcore's
-# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
-# close and retry on its own) — we never expect to see it surface to
-# a transport caller, so it has no httpx counterpart here.
-_HTTPCORE_TO_HTTPX_EXC = {
-    httpcore.ConnectError: httpx.ConnectError,
-    httpcore.ConnectTimeout: httpx.ConnectTimeout,
-    httpcore.LocalProtocolError: httpx.LocalProtocolError,
-    httpcore.NetworkError: httpx.NetworkError,
-    httpcore.PoolTimeout: httpx.PoolTimeout,
-    httpcore.ProtocolError: httpx.ProtocolError,
-    httpcore.ProxyError: httpx.ProxyError,
-    httpcore.ReadError: httpx.ReadError,
-    httpcore.ReadTimeout: httpx.ReadTimeout,
-    httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
-    httpcore.TimeoutException: httpx.TimeoutException,
-    httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
-    httpcore.WriteError: httpx.WriteError,
-    httpcore.WriteTimeout: httpx.WriteTimeout,
-}
-
-
-class _PinnedTransport(httpx.BaseTransport):
-    """Transport that pins every TCP connect to a pre-resolved IP.
-
-    Uses only the public ``httpcore`` and ``httpx`` APIs — no
-    subclassing of ``httpx.HTTPTransport``, no reads of private
-    ``httpcore.ConnectionPool`` attributes, no imports from
-    ``httpx private transport internals``. The URL is passed through unchanged so SNI
-    / vhost work as if httpx had been given the hostname directly;
-    only the TCP destination is pinned, closing the DNS-rebinding
-    TOCTOU between the SSRF check and the connect.
-    """
-
-    def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
-        self._pool = httpcore.ConnectionPool(
-            ssl_context=ssl.create_default_context(),
-            http1=True,
-            http2=http2,
-            network_backend=_PinnedBackend(ip),
-        )
-
-    def __enter__(self):
-        self._pool.__enter__()
-        return self
-
-    def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
-        self._pool.__exit__(exc_type, exc_value, traceback)
-
-    def handle_request(self, request: httpx.Request) -> httpx.Response:
-        httpcore_req = httpcore.Request(
-            method=request.method,
-            url=httpcore.URL(
-                scheme=request.url.raw_scheme,
-                host=request.url.raw_host,
-                port=request.url.port,
-                target=request.url.raw_path,
-            ),
-            headers=request.headers.raw,
-            content=request.stream,
-            extensions=request.extensions,
-        )
-        try:
-            httpcore_resp = self._pool.handle_request(httpcore_req)
-            # Eager materialisation matches the original
-            # ``response.text`` usage in fetch_webpage_content. The
-            # sync pool's stream is a plain Iterable[bytes] despite
-            # the httpcore type hint unioning the async variant.
-            content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
-        except Exception as exc:
-            mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
-            if mapped is not None:
-                raise mapped(str(exc)) from exc
-            raise
-
-        return httpx.Response(
-            status_code=httpcore_resp.status,
-            headers=httpcore_resp.headers,
-            content=content,
-            extensions=httpcore_resp.extensions,
-        )
-
-    def close(self) -> None:
-        self._pool.close()
-
 class BodyTooLargeError(Exception):
     """The server declared a body larger than the hard fetch ceiling."""
 
@@ -285,78 +141,78 @@ class _CappedFetch:
 
 def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
                     max_bytes: int = None) -> "_CappedFetch":
-    """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
+    """Capped streaming GET with SSRF-guarded manual redirects.
 
-    Each hop is resolved once, validated as public, and then the actual TCP
+    The body is streamed and buffering stops at ``max_bytes`` (default: the
-    connection is pinned to that resolved IP. The request URL is left unchanged
+    soft cap), so an oversized resource cannot be pulled into memory or the
-    so Host and TLS SNI keep the original hostname.
+    content cache in full. When Content-Length already declares a body over
+    the hard ceiling, the fetch is refused before any body bytes are read.
     """
     cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
     current = url
     for _ in range(max_redirects + 1):
-        ips = _resolve_public_ips(current)
+        if not _public_http_url(current):
-
+            raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
         # Force identity transfer-encoding. With gzip/deflate the wire bytes
-        # and Content-Length can be a small fraction of the decoded body, so a
+        # (and Content-Length) can be a small fraction of the decoded body, so
-        # tiny compressed response could pass the hard-cap preflight and then
+        # a tiny compressed response could pass the hard-cap preflight and then
-        # expand past the ceiling in one decoded chunk before the streamed cap
+        # expand past the ceiling in a single decoded chunk before the streamed
-        # below can slice it.
+        # cap below can slice it. Identity makes Content-Length the true body
+        # size and keeps each streamed chunk bounded by the network read.
         req_headers = dict(headers or {})
         req_headers["Accept-Encoding"] = "identity"
+        with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
+                          follow_redirects=False) as response:
+            if response.status_code in (301, 302, 303, 307, 308):
+                location = response.headers.get("location")
+                if not location:
+                    return _CappedFetch(response.status_code, response.headers, b"",
+                                        False, None, response.encoding, str(response.url))
+                current = urljoin(str(response.url), location)
+                continue
 
-        with httpx.Client(
+            # A server can ignore the identity request and still return a
-            headers=req_headers,
+            # compressed body; httpx.iter_bytes would then decode it, and a tiny
-            timeout=timeout,
+            # gzip can balloon into one decoded chunk far past the cap before we
-            follow_redirects=False,
+            # slice. Refuse a compressed Content-Encoding so the streamed cap
-            transport=_PinnedTransport(ips[0]),
+            # stays a real memory bound (Content-Length is the compressed wire
-        ) as client:
+            # length here, so the preflight and size metadata are unreliable too).
-            with client.stream("GET", current) as response:
+            enc = (response.headers.get("content-encoding") or "").strip().lower()
-                if response.status_code in (301, 302, 303, 307, 308):
+            if enc and enc != "identity":
-                    location = response.headers.get("location")
+                raise httpx.RequestError(
-                    if not location:
+                    f"Refusing compressed response (Content-Encoding: {enc}) after "
-                        return _CappedFetch(response.status_code, response.headers, b"",
+                    "requesting identity: cannot bound decoded body size",
-                                            False, None, response.encoding, str(response.url))
+                    request=httpx.Request("GET", current),
-                    current = urljoin(str(response.url), location)
+                )
-                    continue
 
-                # A server can ignore the identity request and still return a
+            declared = None
-                # compressed body; httpx.iter_bytes would then decode it, and a
+            raw_len = response.headers.get("content-length")
-                # tiny gzip can balloon into one decoded chunk far past the cap.
+            if raw_len and raw_len.isdigit():
-                # Refuse compressed Content-Encoding so the streamed cap stays
+                declared = int(raw_len)
-                # a real memory bound.
+            # Refuse before buffering anything when the server already tells
-                enc = (response.headers.get("content-encoding") or "").strip().lower()
+            # us the body exceeds the absolute ceiling (Content-Length is wire
-                if enc and enc != "identity":
+            # bytes; the decompressed body can only be larger).
-                    raise httpx.RequestError(
+            if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
-                        f"Refusing compressed response (Content-Encoding: {enc}) after "
+                raise BodyTooLargeError(current, declared)
-                        "requesting identity: cannot bound decoded body size",
-                        request=httpx.Request("GET", current),
-                    )
-
-                declared = None
-                raw_len = response.headers.get("content-length")
-                if raw_len and raw_len.isdigit():
-                    declared = int(raw_len)
-
-                if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
-                    raise BodyTooLargeError(current, declared)
-
-                chunks = []
-                read = 0
-                truncated = False
-                for chunk in response.iter_bytes():
-                    read += len(chunk)
-                    if read > cap:
-                        keep = cap - (read - len(chunk))
-                        if keep > 0:
-                            chunks.append(chunk[:keep])
-                        truncated = True
-                        break
-                    chunks.append(chunk)
-
-                return _CappedFetch(response.status_code, response.headers,
-                                    b"".join(chunks), truncated, declared,
-                                    response.encoding, str(response.url))
 
+            chunks = []
+            read = 0
+            truncated = False
+            # We requested identity above, so iter_bytes yields the raw body in
+            # network-read-sized chunks (no decompression expansion); the cap
+            # therefore bounds what we actually buffer.
+            for chunk in response.iter_bytes():
+                read += len(chunk)
+                if read > cap:
+                    keep = cap - (read - len(chunk))
+                    if keep > 0:
+                        chunks.append(chunk[:keep])
+                    truncated = True
+                    break
+                chunks.append(chunk)
+            return _CappedFetch(response.status_code, response.headers,
+                                b"".join(chunks), truncated, declared,
+                                response.encoding, str(response.url))
     raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
 
 # PDF extraction (optional dependency)

services/search/query.py

@@ -34,14 +34,8 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
     cleaned = query
     if qtype:
         cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
-    # Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
+    for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
-    # class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
+        entities["names"].append(token)
-    # "São" (shredded). Keep the ASCII behaviour — the word boundary already
-    # excludes camelCase mid-word capitals — by requiring an all-alphabetic
-    # token of length > 1 whose first character is uppercase.
-    for token in re.findall(r"\b\w+\b", cleaned):
-        if len(token) > 1 and token[0].isupper() and token.isalpha():
-            entities["names"].append(token)
     for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
         entities["dates"].append(year)
     month_day_year = re.findall(

src/action_intents.py

@@ -92,21 +92,8 @@ _ROUTING_PATTERNS: tuple[tuple[str, str, Pattern[str]], ...] = tuple(
 
         # Deep research jobs, not quick conceptual mentions of research.
         ("web", "explicit web search request", rf"{_PLEASE}(?:do|run|use|perform|make)\s+(?:a\s+)?(?:web\s+search|search\s+the\s+web)\b.+"),
-        ("web", "generic search request", rf"{_PLEASE}search\s+(?!(?:my\s+)?(?:chats?|history|sessions?|notes?|todos?|emails?|mail|inbox|documents?|docs|gallery|images?|files?)\b).+"),
+        ("web", "web lookup imperative request", rf"{_PLEASE}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"),
-        ("web", "web lookup imperative request", rf"{_PLEASE}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google(?:\s+it)?)\b.*"),
+        ("web", "assistant web lookup request", rf"{_ACTION_QUESTION}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google)\b.+"),
-        ("web", "short web lookup follow-up", rf"{_PLEASE}(?:just\s+)?(?:look\s+it\s+up|look\s+up|search\s+(?:online|web|now)|search\s+it)\b\s*$"),
-        ("web", "assistant short web lookup request", rf"{_ACTION_QUESTION}(?:search|look\s+up|google)(?:\s+(?:online|web|now|it))?\b.*"),
-        ("web", "assistant web lookup request", rf"{_ACTION_QUESTION}(?:web\s+search|search\s+the\s+web|search\s+online|look\s+up|google(?:\s+it)?)\b.*"),
-        ("web", "assistant weather check request", rf"{_ACTION_QUESTION}(?:check|find|get|look\s+up)\b.{{0,100}}\b(?:weather|forecast)\b.*"),
-        ("web", "news lookup request", r"\b(?:news|headlines)\s+(?:in|from|about|for)\s+[\w\s.-]{2,80}\??\s*$"),
-        ("web", "forecast lookup request", r"\b(?:hourly|daily|weekly|local)\s+(?:weather\s+)?forecast\b|\b(?:weather\s+)?forecast\s+(?:for|today|tomorrow|now|hourly)\b"),
-        ("web", "weather lookup request", r"\bweather\b.{0,80}\b(?:hourly|rain|raining|rin|today|tomorrow|update|current|now)\b|\b(?:hourly|rain|raining|rin)\b.{0,80}\bweather\b"),
-        ("web", "rain lookup request", r"\b(?:hourly|daily|weekly|local|today|tomorrow|current|now|update)\b.{0,100}\b(?:rain|raining|rainy|precipitation|showers?)\b|\b(?:rain|raining|rainy|precipitation|showers?)\b.{0,100}\b(?:hourly|daily|weekly|local|today|tomorrow|current|now|update|in|for|at)\b"),
-        ("web", "bare weather lookup request", r"\b(?:weather|forecast)\s+(?:in|for|at)?\s*[\w\s.-]{2,80}\??\s*$|\b[\w\s.-]{2,80}\s+(?:weather|forecast)\??\s*$"),
-        ("web", "latest info lookup request", r"\b(?:latest|current|newest|recent|up(?: |-)?to(?: |-)?date)\s+(?:info|information|updates?|details?|developments?)\s+(?:on|about|for|in)\s+[\w\s.,:'\"/-]{2,120}\??\s*$"),
-        ("web", "current/latest lookup request", r"\b(?:current|latest|today'?s?|right\s+now|live|online)\b.{0,120}\b(?:rate|price|news|weather|forecast|score|exchange|market|status)\b"),
-        ("web", "rate/price/news lookup request", r"\b(?:rate|rates|price|prices|news|weather|forecast|score|exchange|currency|market)\b.{0,120}\b(?:now|today|current|latest|online|live|search|look\s+up|find)\b"),
-        ("web", "conversion-rate lookup request", r"\b(?:convert|conversion|exchange)\b.{0,120}\b(?:rate|rates|currency|currencies|price|prices)\b"),
         ("research", "deep research imperative request", rf"{_PLEASE}(?:research|deep\s+dive|look\s+into|investigate)\s+.+"),
         ("research", "assistant deep research request", rf"{_ACTION_QUESTION}(?:research|do\s+research|deep\s+dive|look\s+into|investigate)\s+.+"),
 

src/agent_tools/__init__.py

@@ -14,7 +14,6 @@ Sub-modules:
 import logging
 from collections import namedtuple
 
-from src.tool_security import BUILTIN_EMAIL_TOOLS
 from src.tool_utils import _truncate, get_mcp_manager, set_mcp_manager
 
 logger = logging.getLogger(__name__)
@@ -87,10 +86,9 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi
              "manage_endpoints", "manage_mcp", "manage_webhooks",
              "manage_tokens", "manage_documents", "manage_settings",
              "manage_notes", "manage_calendar",
-             "resolve_contact", "manage_contact",
+             "resolve_contact", "manage_contact", "list_email_accounts", "send_email", "list_emails",
-             # Email tool names come from BUILTIN_EMAIL_TOOLS (unioned below)
+             "read_email", "reply_to_email", "bulk_email", "archive_email",
-             # so the fence regex, dispatch, and non-admin blocklist all cover
+             "delete_email", "mark_email_read",
-             # the same set.
              # Cookbook tools (LLM serving + downloads). Without these
              # entries, native function calls to e.g. list_served_models
              # are rejected as "Unknown function call" before reaching
@@ -107,7 +105,7 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi
              # Generic loopback to any UI-button endpoint (cookbook,
              # gallery, email folders, etc.) — agent uses this when
              # there's no named tool wrapper for the action.
-             "app_api"} | BUILTIN_EMAIL_TOOLS
+             "app_api"}
 
 ToolBlock = namedtuple("ToolBlock", ["tool_type", "content"])
 

src/agent_tools/admin_tools.py

@@ -14,7 +14,6 @@ import logging
 from typing import Optional, Dict
 
 from src.tool_utils import get_mcp_manager, _parse_tool_args
-from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
@@ -707,14 +706,7 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
                 "tasks": ["manage_tasks"],
                 "notes": ["manage_notes"],
                 "calendar": ["manage_calendar"],
-                # The full built-in email tool set, in BOTH spellings: the
+                "email": ["mcp__email__list_emails", "mcp__email__read_email", "mcp__email__send_email"],
-                # qualified mcp__email__* names drive MCP schema hiding, the
-                # bare names drive function-schema hiding, and the runtime
-                # gate accepts either — deriving from BUILTIN_EMAIL_TOOLS
-                # keeps the toggle covering every tool the email server
-                # exposes instead of a hand-picked subset.
-                "email": sorted(BUILTIN_EMAIL_TOOLS)
-                         + [f"mcp__email__{t}" for t in sorted(BUILTIN_EMAIL_TOOLS)],
                 "research": ["web_search", "web_fetch"],  # research is a per-request flag, not a tool (closest analog)
             }
 

src/agent_tools/document_tools.py

@@ -130,70 +130,15 @@ def _looks_like_email_document(text: str = "", title: str = "") -> bool:
         return True
     return bool(_re.search(r"(?im)^To:\s*", s) and _re.search(r"(?im)^Subject:\s*", s))
 
-def _split_email_header_body(text: str) -> tuple[str, str]:
-    if "\n---\n" in (text or ""):
-        header, body = (text or "").split("\n---\n", 1)
-        return header.rstrip(), body.strip()
-    return (text or "").strip(), ""
-
-def _split_email_reply_history(body: str) -> tuple[str, str]:
-    """Split draft body from quoted/original email history.
-
-    Email reply docs keep the original thread below the user's new reply. Models
-    often rewrite only the fresh reply body; this helper keeps the historical
-    block from being wiped when update_document/edit_document replaces content.
-    """
-    text = body or ""
-    literal = "---------- Previous message ----------"
-    literal_idx = text.find(literal)
-    if literal_idx >= 0:
-        return text[:literal_idx].strip(), text[literal_idx:].strip()
-    patterns = [
-        r"(?m)^On .+ wrote:\s*$",
-        r"(?m)^> .+",
-    ]
-    starts = []
-    for pat in patterns:
-        m = re.search(pat, text)
-        if m:
-            starts.append(m.start())
-    if not starts:
-        return text.strip(), ""
-    idx = min(starts)
-    return text[:idx].strip(), text[idx:].strip()
-
-def _merge_email_headers(old_header: str, new_header: str) -> str:
-    """Preserve routing/threading metadata if a model omits it."""
-    protected = (
-        "In-Reply-To", "References", "X-Source-UID", "X-Source-Folder",
-        "X-Attachments", "X-Forward-Attachments",
-    )
-    lines = [l for l in (new_header or "").splitlines() if l.strip()]
-    present = {l.split(":", 1)[0].strip().lower() for l in lines if ":" in l}
-    for old_line in (old_header or "").splitlines():
-        if ":" not in old_line:
-            continue
-        key = old_line.split(":", 1)[0].strip()
-        if key in protected and key.lower() not in present:
-            lines.append(old_line)
-            present.add(key.lower())
-    return "\n".join(lines).rstrip()
-
 def _coerce_email_document_content(existing: str, incoming: str) -> str:
     """Keep email docs in the To/Subject/---/body shape even if a model writes
     only the body or dumps header labels without the separator."""
     import re as _re
     old = existing or ""
     new = (incoming or "").strip()
-    old_header, old_body = _split_email_header_body(old)
-    _, old_history = _split_email_reply_history(old_body)
     if "\n---\n" in new:
-        new_header, new_body = _split_email_header_body(new)
+        return new
-        new_own, new_history = _split_email_reply_history(new_body)
+    header = old.split("\n---\n", 1)[0] if "\n---\n" in old else "To: \nSubject: "
-        if old_history and not new_history:
-            new_body = (new_own + "\n\n" + old_history).strip()
-        return _merge_email_headers(old_header, new_header).rstrip() + "\n---\n" + new_body
-    header = old_header if old_header else "To: \nSubject: "
     if _looks_like_email_document(new):
         lines = new.splitlines()
         last_header_idx = -1
@@ -207,9 +152,6 @@ def _coerce_email_document_content(existing: str, incoming: str) -> str:
         body = "\n".join(body_lines).strip()
     else:
         body = new
-    _, incoming_history = _split_email_reply_history(body)
-    if old_history and not incoming_history:
-        body = (body.strip() + "\n\n" + old_history).strip()
     return header.rstrip() + "\n---\n" + body
 
 def parse_edit_blocks(content: str) -> list:
@@ -243,71 +185,6 @@ def parse_suggest_blocks(content: str) -> list:
     return suggestions
 
 
-def _pdf_source_upload_id(content: str) -> Optional[str]:
-    try:
-        from src.pdf_form_doc import find_source_upload_id
-        return find_source_upload_id(content or "")
-    except Exception:
-        return None
-
-
-def _strip_pdf_editor_markers(content: str) -> str:
-    """Turn a PDF-wrapper markdown doc into ordinary editable markdown.
-
-    PDF docs use hidden HTML comments for source-upload links, form fields, and
-    page annotations. Those comments are necessary for rendering/exporting the
-    original PDF, but they make a derived AI text edit keep showing the original
-    PDF preview. Remove only the editor plumbing and keep the readable text.
-    """
-    text = content or ""
-    text = re.sub(r'(?im)^\s*<!--\s*pdf(?:_form)?_source\s+[^>]*-->\s*\n*', '', text)
-    text = re.sub(r'\s*<!--\s*field=[^>]*-->', '', text)
-    text = re.sub(r'\s*<!--\s*annotation\s+[^>]*-->', '', text)
-    return text.strip()
-
-
-def _create_pdf_text_derivative(db, *, source_doc, content: str, owner: Optional[str], summary: str) -> dict:
-    import uuid
-    from src.database import Document, DocumentVersion
-
-    clean = _strip_pdf_editor_markers(content)
-    title_base = (getattr(source_doc, "title", None) or "PDF").strip()
-    title = title_base if title_base.lower().endswith("edited") else f"{title_base} edited"
-    doc_id = str(uuid.uuid4())
-    ver_id = str(uuid.uuid4())
-    new_doc = Document(
-        id=doc_id,
-        session_id=getattr(source_doc, "session_id", None),
-        title=title,
-        language="markdown",
-        current_content=clean,
-        version_count=1,
-        is_active=True,
-        owner=owner if owner is not None else getattr(source_doc, "owner", None),
-    )
-    ver = DocumentVersion(
-        id=ver_id,
-        document_id=doc_id,
-        version_number=1,
-        content=clean,
-        summary=summary,
-        source="ai",
-    )
-    db.add(new_doc)
-    db.add(ver)
-    db.commit()
-    set_active_document(doc_id)
-    return {
-        "action": "create",
-        "doc_id": doc_id,
-        "title": title,
-        "language": "markdown",
-        "content": clean,
-        "version": 1,
-        "source_doc_id": getattr(source_doc, "id", None),
-    }
-
-
 class CreateDocumentTool:
     async def execute(self, content: str, ctx: dict) -> dict:
         """Create a new document. Supports two formats:
@@ -455,15 +332,6 @@ class UpdateDocumentTool:
             if is_email_doc:
                 doc.language = "email"
 
-            if not is_email_doc and _pdf_source_upload_id(doc.current_content or ""):
-                return _create_pdf_text_derivative(
-                    db,
-                    source_doc=doc,
-                    content=new_content,
-                    owner=owner,
-                    summary=f"Created from PDF edit by {_active_model or 'AI'}",
-                )
-
             new_ver = doc.version_count + 1
             ver = DocumentVersion(
                 id=str(uuid.uuid4()),
@@ -521,42 +389,6 @@ class EditDocumentTool:
             if not doc:
                 return {"error": "No documents exist to edit"}
 
-            is_email_doc = doc.language == "email" or _looks_like_email_document(doc.current_content or "", doc.title or "")
-            blank_find_edits = [e for e in edits if not (e.get("find") or "").strip()]
-            if blank_find_edits:
-                if is_email_doc:
-                    replacement_body = (blank_find_edits[0].get("replace") or "").strip()
-                    if not replacement_body:
-                        return {"error": "No edits applied — blank FIND block had no replacement text"}
-                    updated_content = _coerce_email_document_content(doc.current_content or "", replacement_body)
-                    applied = 1
-                    skipped = max(0, len(edits) - 1)
-                    doc.language = "email"
-                    new_ver = doc.version_count + 1
-                    ver = DocumentVersion(
-                        id=str(uuid.uuid4()),
-                        document_id=target_id,
-                        version_number=new_ver,
-                        content=updated_content,
-                        summary=f"Edited email body by {_active_model or 'AI'}",
-                        source="ai",
-                    )
-                    doc.current_content = updated_content
-                    doc.version_count = new_ver
-                    db.add(ver)
-                    db.commit()
-                    return {
-                        "action": "edit",
-                        "doc_id": target_id,
-                        "title": doc.title,
-                        "language": doc.language,
-                        "content": updated_content,
-                        "version": new_ver,
-                        "applied": applied,
-                        "skipped": skipped,
-                    }
-                return {"error": "No edits applied — FIND text cannot be blank"}
-
             updated_content = doc.current_content
             applied = 0
             skipped = 0
@@ -584,15 +416,6 @@ class EditDocumentTool:
             if applied == 0:
                 return {"error": f"No edits applied — none of the FIND blocks matched the document content (skipped {skipped})"}
 
-            if _pdf_source_upload_id(doc.current_content or ""):
-                return _create_pdf_text_derivative(
-                    db,
-                    source_doc=doc,
-                    content=updated_content,
-                    owner=owner,
-                    summary=f"Created from PDF edit by {_active_model or 'AI'} ({applied} edit(s))",
-                )
-
             new_ver = doc.version_count + 1
             ver = DocumentVersion(
                 id=str(uuid.uuid4()),

src/agent_tools/filesystem_tools.py

@@ -186,21 +186,6 @@ class WriteFileTool:
         lines = content.split("\n", 1)
         raw_path = lines[0].strip()
         body = lines[1] if len(lines) > 1 else ""
-        # Decode JSON-object args (the fenced inline-args shape
-        # ```write_file {"path": "...", "content": "..."}```), matching
-        # ReadFileTool above. Without this the whole JSON string becomes the
-        # path and the file is written under a garbage name. This is the live
-        # path: there is no filesystem MCP server, so write_file always runs
-        # here via _direct_fallback, not through _build_mcp_args.
-        _stripped = content.strip()
-        if _stripped.startswith("{"):
-            try:
-                _a = json.loads(_stripped)
-                if isinstance(_a, dict) and "path" in _a:
-                    raw_path = str(_a.get("path", "")).strip()
-                    body = str(_a.get("content", ""))
-            except (json.JSONDecodeError, TypeError, ValueError):
-                pass
         try:
             path = _resolve_tool_path(raw_path)
         except ValueError as e:
@@ -281,13 +266,7 @@ class LsTool:
 
 class GlobTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
-            _SENSITIVE_BASENAMES,
-            _is_sensitive_path,
-            _resolve_tool_path,
-            _resolve_search_root,
-            _truncate,
-        )
         args = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -309,30 +288,11 @@ class GlobTool:
             base = os.path.abspath(root)
             if not os.path.isdir(base):
                 return None, f"glob: {root}: not a directory"
-            rbase = os.path.realpath(base)
             norm_pat = pattern.replace("\\", "/")
             # Fast path: literal pattern (no wildcards) → direct path lookup.
             if not any(c in norm_pat for c in "*?["):
-                cand = os.path.realpath(os.path.join(base, norm_pat))
+                cand = os.path.normpath(os.path.join(base, norm_pat))
-                # Keep the literal lookup inside the search root. os.path.join
+                if os.path.exists(cand):
-                # lets an absolute pattern (or one containing ../) escape `base`,
-                # which would turn glob into an existence/path oracle for
-                # arbitrary host files — bypassing the workspace/allowlist
-                # confinement that _resolve_search_root applies to the root.
-                # An escaping literal falls through to the walk, which only ever
-                # yields paths under base.
-                nbase = os.path.normcase(rbase)
-                try:
-                    inside = cand == rbase or os.path.commonpath(
-                        [os.path.normcase(cand), nbase]
-                    ) == nbase
-                except ValueError:
-                    inside = False
-                # A literal that names a deny-listed sensitive file (.env,
-                # .ssh/id_rsa, …) falls through to the walk, which skips it —
-                # otherwise glob would surface secret paths that read_file /
-                # grep already refuse to touch.
-                if inside and os.path.exists(cand) and not _is_sensitive_path(cand):
                     return [cand], None
                 # Literal not at exact path — fall through to walk so
                 # e.g. "foo.py" still matches at any depth (like rglob).
@@ -344,20 +304,11 @@ class GlobTool:
                 for dp, dns, fns in os.walk(base):
                     # Prune skipped dirs before descending (unlike rglob which
                     # descends first then filters — fatal on large node_modules).
-                    # Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
+                    dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
-                    # never enumerates the keys/tokens inside them.
-                    dns[:] = [
-                        d for d in dns
-                        if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
-                    ]
                     for name in fns + dns:
                         full = os.path.join(dp, name)
                         rel = os.path.relpath(full, base).replace(os.sep, "/")
                         if regex.fullmatch(rel) or regex.fullmatch(name):
-                            # Skip deny-listed sensitive files (.env, id_rsa,
-                            # known_hosts, …) the same way grep does.
-                            if _is_sensitive_path(os.path.realpath(full)):
-                                continue
                             try:
                                 mtime = os.stat(full).st_mtime
                             except OSError:
@@ -382,13 +333,7 @@ class GlobTool:
 
 class GrepTool:
     async def execute(self, content: str, ctx: dict) -> dict:
-        from src.tool_execution import (
+        from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
-            _SENSITIVE_FILE_PATTERNS,
-            _is_sensitive_path,
-            _resolve_tool_path,
-            _resolve_search_root,
-            _truncate,
-        )
         args: Dict[str, Any] = {}
         _s = (content or "").strip()
         if _s.startswith("{"):
@@ -424,12 +369,6 @@ class GrepTool:
                     cmd.append("--ignore-case")
                 if glob_pat:
                     cmd += ["--glob", glob_pat]
-                # --iglob (not --glob) so the exclusion is case-insensitive:
-                # on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
-                # resolve to the same secret as their lowercase forms, and the
-                # Python fallback below already folds case via _is_sensitive_path.
-                for _pat in _SENSITIVE_FILE_PATTERNS:
-                    cmd += ["--iglob", f"!*{_pat}*"]
                 for _d in _CODENAV_SKIP_DIRS:
                     cmd += ["--glob", f"!**/{_d}/**"]
                 cmd += ["--regexp", pattern, root]
@@ -460,8 +399,6 @@ class GrepTool:
             for fp in file_iter:
                 if len(hits) >= max_hits:
                     break
-                if _is_sensitive_path(os.path.realpath(fp)):
-                    continue
                 try:
                     with open(fp, "r", encoding="utf-8", errors="strict") as f:
                         for i, line in enumerate(f, 1):

src/agent_tools/session_tools.py

@@ -104,8 +104,6 @@ async def list_sessions(content: str, session_id: Optional[str] = None, owner: O
         sessions = _session_manager.get_sessions_for_user(owner)
         rows = []
         for sid, sess in sessions.items():
-            if (sess.name or "").startswith("SFT trace batch"):
-                continue
             if keyword and keyword not in (sess.name or "").lower():
                 continue
             db_row = db_rows.get(sid)
@@ -184,12 +182,8 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner:
     if not sess:
         return {"error": f"Session '{target_sid}' not found"}
 
-    # Owner-scope: reject access to another user's session. When the caller is
+    # Owner-scope: reject access to another user's session
-    # authenticated, a null-owner (legacy / auth-was-off) session is not theirs
+    if owner and getattr(sess, "owner", None) and sess.owner != owner:
-    # either — list_sessions (get_sessions_for_user) and manage_session already
-    # exclude those, so treating it as reachable here let an authenticated agent
-    # read/write a session the other tools hide. Require an exact owner match.
-    if owner and getattr(sess, "owner", None) != owner:
         return {"error": f"Session '{target_sid}' not found"}
 
     if not message:
@@ -198,25 +192,6 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner:
     try:
         # Build context from session history
         context = sess.get_context_messages()
-        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
-        model = str(getattr(sess, "model", "") or "")
-        if model == "fixture-tool-model" or "host.docker.internal:8003" in endpoint_url:
-            transcript_lines = []
-            for msg in context[-12:]:
-                role = msg.get("role", "unknown")
-                text = (msg.get("content") or "").strip()
-                if text:
-                    transcript_lines.append(f"{role}: {text}")
-            transcript = "\n".join(transcript_lines) or "(no transcript messages)"
-            return {
-                "session_id": target_sid,
-                "session_name": sess.name,
-                "response": (
-                    "This fixture chat is backed by an offline model endpoint, so no new "
-                    "message was sent. Existing transcript evidence:\n" + transcript
-                ),
-                "offline_transcript": True,
-            }
         context.append({"role": "user", "content": message})
 
         response = await llm_call_async(

src/agent_tools/web_tools.py

@@ -1,5 +1,4 @@
 import asyncio
-import inspect
 import json
 from typing import Dict, Any
 
@@ -110,20 +109,8 @@ class WebFetchTool:
             url = "https://" + url
         loop = asyncio.get_running_loop()
         try:
-            def _fetch():
-                kwargs = {"timeout": 10}
-                try:
-                    sig = inspect.signature(fetch_webpage_content)
-                    if "max_bytes" in sig.parameters:
-                        kwargs["max_bytes"] = max_bytes
-                except (TypeError, ValueError):
-                    # Some deployed/test shims may not expose a signature.
-                    # Prefer compatibility over failing the whole fetch.
-                    pass
-                return fetch_webpage_content(url, **kwargs)
-
             result = await asyncio.wait_for(
-                loop.run_in_executor(None, _fetch),
+                loop.run_in_executor(None, lambda: fetch_webpage_content(url, timeout=10, max_bytes=max_bytes)),
                 timeout=30,
             )
         except asyncio.TimeoutError:

src/ai_interaction.py

@@ -433,23 +433,13 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner
             return {"error": "Search needs line 2: query"}
         query = lines[1].strip()
         memories = _memory_manager.load(owner=owner)
-        query_lower = query.lower()
-        exact_results = [m for m in memories if query_lower in (m.get("text", "").lower())]
 
         if hasattr(_memory_manager, 'get_relevant_memories'):
-            vector_results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+            results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
         else:
-            vector_results = []
+            # Fallback: simple text search
-        seen = set()
+            query_lower = query.lower()
-        results = []
+            results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
-        for m in [*exact_results, *vector_results]:
-            mid = m.get("id")
-            if mid in seen:
-                continue
-            seen.add(mid)
-            results.append(m)
-            if len(results) >= 20:
-                break
 
         if not results:
             return {"results": f"No memories found matching '{query}'."}

src/builtin_actions.py

@@ -7,7 +7,6 @@ scheduler without needing an LLM call.
 
 import logging
 import os
-import json
 from datetime import datetime
 from typing import Tuple
 
@@ -15,7 +14,6 @@ from src.auth_helpers import owner_filter
 from core.platform_compat import IS_WINDOWS, find_bash
 from core.constants import internal_api_base
 from src.constants import DATA_DIR, DEEP_RESEARCH_DIR, TIDY_CALENDAR_STATE_FILE, EMAIL_URGENCY_CACHE_DIR, COOKBOOK_STATE_FILE
-from src.interactive_gate import wait_for_interactive_quiet
 
 logger = logging.getLogger(__name__)
 
@@ -147,7 +145,6 @@ async def action_consolidate_memory(owner: str, **kwargs) -> Tuple[str, bool]:
                     "\"drop\":[{\"id\":\"existing id\",\"reason\":\"short reason\"}]}\n\n"
                     f"MEMORIES:\n{json.dumps(items, ensure_ascii=False)}"
                 )
-                await wait_for_interactive_quiet("memory consolidation action")
                 raw = await llm_call_async_with_fallback(
                     candidates,
                     messages=[{"role": "user", "content": prompt}],
@@ -500,48 +497,11 @@ def _result_has_work(result: str | None) -> bool:
     return True
 
 
-def _result_is_config_error(result: str | None) -> bool:
-    if not isinstance(result, str):
-        return False
-    low = result.lower()
-    return (
-        "no model configured" in low
-        or "no model endpoint configured" in low
-        or "no llm endpoint available" in low
-    )
-
-
-def _email_task_account_id(kwargs) -> str | None:
-    prompt = (kwargs.get("prompt") or "").strip()
-    if not prompt:
-        return None
-    try:
-        data = json.loads(prompt)
-        if isinstance(data, dict):
-            val = data.get("account_id") or data.get("email_account_id")
-            return str(val).strip() or None
-    except Exception:
-        pass
-    for line in prompt.splitlines():
-        if "=" not in line:
-            continue
-        key, val = line.split("=", 1)
-        if key.strip().lower() in {"account_id", "email_account_id"}:
-            return val.strip() or None
-    return None
-
-
 async def action_summarize_emails(owner: str, **kwargs) -> Tuple[str, bool]:
     """Run one pass of email summary background processing."""
     try:
         from routes.email_pollers import _run_auto_summarize_once
-        result = await _run_auto_summarize_once(
+        result = await _run_auto_summarize_once(do_summary=True, do_reply=False)
-            do_summary=True,
-            do_reply=False,
-            account_id=_email_task_account_id(kwargs),
-        )
-        if _result_is_config_error(result):
-            return result, False
         if not _result_has_work(result):
             raise TaskNoop(f"summarize: {result or 'no new emails'}")
         return result, True
@@ -557,12 +517,9 @@ async def action_draft_email_replies(owner: str, **kwargs) -> Tuple[str, bool]:
         result = await _run_auto_summarize_once(
             do_summary=False,
             do_reply=True,
-            account_id=_email_task_account_id(kwargs),
             days_back=7,
             progress_cb=kwargs.get("progress_cb"),
         )
-        if _result_is_config_error(result):
-            return result, False
         if not _result_has_work(result):
             raise TaskNoop(f"draft replies: {result or 'no new emails'}")
         return result, True
@@ -571,250 +528,6 @@ async def action_draft_email_replies(owner: str, **kwargs) -> Tuple[str, bool]:
         return str(e), False
 
 
-async def action_email_auto_translate(owner: str, **kwargs) -> Tuple[str, bool]:
-    """Detect recent foreign-language emails and cache translated text.
-
-    The reader still shows the original body; it simply checks this cache
-    before calling the LLM on demand. Keep the scheduled pass deliberately
-    small so translation never turns into a mailbox-wide background crawl.
-    """
-    try:
-        import email as _email_mod
-        import json as _json
-        import re as _re
-        import sqlite3 as _sql3
-        from datetime import datetime as _dt, timedelta as _td
-
-        from core.database import EmailAccount as _EA, SessionLocal as _SL
-        from routes.email_helpers import (
-            SCHEDULED_DB,
-            _decode_header,
-            _email_cache_owner_clause,
-            _extract_reply,
-            _extract_text,
-            _imap_connect,
-            email_translation_body_hash,
-        )
-        from src.settings import load_settings
-        from src.task_endpoint import task_llm_call_async
-
-        settings = load_settings()
-        if not settings.get("email_auto_translate", False):
-            raise TaskNoop("email auto-translate is disabled")
-
-        target_language = (settings.get("email_translate_language") or "English").strip() or "English"
-        account_id = _email_task_account_id(kwargs)
-        days_back = 7
-        max_process = 5
-        try:
-            data = _json.loads((kwargs.get("prompt") or "").strip() or "{}")
-            if isinstance(data, dict):
-                days_back = max(1, min(30, int(data.get("days_back") or days_back)))
-                max_process = max(1, min(20, int(data.get("max_process") or max_process)))
-        except Exception:
-            pass
-
-        db = _SL()
-        try:
-            from sqlalchemy import and_ as _and, or_ as _or
-            q = db.query(_EA).filter(_EA.enabled == True)  # noqa: E712
-            if owner:
-                unowned = _or(_EA.owner == None, _EA.owner == "")  # noqa: E711
-                same_mailbox = _or(_EA.imap_user == owner, _EA.from_address == owner)
-                q = q.filter(_or(_EA.owner == owner, _and(unowned, same_mailbox)))
-            if account_id:
-                q = q.filter(_EA.id == account_id)
-            accounts = q.all()
-        finally:
-            db.close()
-        if not accounts:
-            raise TaskNoop("no email accounts configured")
-
-        def _cached(body_hash: str) -> bool:
-            c = _sql3.connect(SCHEDULED_DB)
-            try:
-                owner_clause, owner_params = _email_cache_owner_clause(owner)
-                row = c.execute(
-                    f"SELECT 1 FROM email_translations "
-                    f"WHERE body_hash = ? AND target_language = ? AND {owner_clause} LIMIT 1",
-                    (body_hash, target_language, *owner_params),
-                ).fetchone()
-                return bool(row)
-            finally:
-                c.close()
-
-        def _store(
-            body_hash: str,
-            *,
-            uid: str,
-            folder: str,
-            subject: str,
-            sender: str,
-            translation: str,
-            same_language: bool,
-            model_used: str,
-        ) -> None:
-            c = _sql3.connect(SCHEDULED_DB)
-            try:
-                c.execute("""
-                    INSERT OR REPLACE INTO email_translations
-                    (body_hash, owner, target_language, uid, folder, subject, sender,
-                     translation, same_language, model_used, created_at)
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-                """, (
-                    body_hash, owner, target_language, uid, folder, subject, sender,
-                    translation, 1 if same_language else 0, model_used, _dt.utcnow().isoformat(),
-                ))
-                c.commit()
-            finally:
-                c.close()
-
-        async def _translate(body: str, subject: str, sender: str) -> tuple[str, bool]:
-            content = await task_llm_call_async(
-                [
-                    {
-                        "role": "system",
-                        "content": (
-                            "You translate emails faithfully. Preserve meaning, names, dates, money, addresses, "
-                            "bullet structure, and tone. Do not summarize or answer the email. "
-                            "Output only the translation between <<<TRANSLATION>>> and <<<END>>>. "
-                            "If the email is already primarily in the target language, output exactly "
-                            "<<<SAME_LANGUAGE>>>."
-                        ),
-                    },
-                    {
-                        "role": "user",
-                        "content": (
-                            f"Target language: {target_language}\n\n"
-                            f"From: {sender}\nSubject: {subject}\n\n{body[:16000]}\n\n"
-                            "Translate the email unless it is already primarily in the target language.\n"
-                            "Return only:\n<<<TRANSLATION>>>\ntranslated text\n<<<END>>>"
-                        ),
-                    },
-                ],
-                owner=owner,
-                temperature=0.2,
-                max_tokens=8192,
-                timeout=180,
-            )
-            content = (content or "").strip()
-            content = _extract_reply(content)
-            if "<<<SAME_LANGUAGE>>>" in content:
-                return "", True
-            marker = _re.search(r"<<<TRANSLATION>>>\s*(.*?)\s*<<<END>>>", content, _re.S | _re.I)
-            if marker:
-                content = marker.group(1).strip()
-            else:
-                content = _re.sub(r"^\s*<<<TRANSLATION>>>\s*", "", content, flags=_re.I).strip()
-                content = _re.sub(r"\s*<<<END>>>\s*$", "", content, flags=_re.I).strip()
-            return content, False
-
-        since = (_dt.utcnow() - _td(days=days_back)).strftime("%d-%b-%Y")
-        examined = 0
-        cached = 0
-        translated = 0
-        same_language = 0
-        skipped = 0
-        failures = 0
-        processed = 0
-
-        for acct in accounts:
-            if processed >= max_process:
-                break
-            imap = None
-            try:
-                imap = _imap_connect(acct.id, owner=owner)
-                imap.select("INBOX", readonly=True)
-                status, data = imap.uid("SEARCH", None, f'(SINCE {since})')
-                if status != "OK" or not data or not data[0]:
-                    continue
-                uids = list(reversed(data[0].split()))[:50]
-                for uid_b in uids:
-                    if processed >= max_process:
-                        break
-                    uid = uid_b.decode("utf-8", errors="ignore") if isinstance(uid_b, bytes) else str(uid_b)
-                    status, msg_data = imap.uid("FETCH", uid, "(RFC822)")
-                    if status != "OK" or not msg_data:
-                        continue
-                    raw = None
-                    for part in msg_data:
-                        if isinstance(part, tuple) and len(part) > 1:
-                            raw = part[1]
-                            break
-                    if not raw:
-                        continue
-                    msg = _email_mod.message_from_bytes(raw)
-                    subject = _decode_header(msg.get("Subject", ""))
-                    sender = _decode_header(msg.get("From", ""))
-                    body = (_extract_text(msg) or "").strip()
-                    examined += 1
-                    if len(body) < 80:
-                        skipped += 1
-                        continue
-                    body_hash = email_translation_body_hash(body)
-                    if _cached(body_hash):
-                        cached += 1
-                        continue
-                    translation, is_same_language = await _translate(body, subject, sender)
-                    if is_same_language:
-                        _store(
-                            body_hash,
-                            uid=uid,
-                            folder="INBOX",
-                            subject=subject,
-                            sender=sender,
-                            translation="",
-                            same_language=True,
-                            model_used="background-task",
-                        )
-                        same_language += 1
-                        processed += 1
-                        continue
-                    if not translation:
-                        failures += 1
-                        continue
-                    _store(
-                        body_hash,
-                        uid=uid,
-                        folder="INBOX",
-                        subject=subject,
-                        sender=sender,
-                        translation=translation,
-                        same_language=False,
-                        model_used="background-task",
-                    )
-                    translated += 1
-                    processed += 1
-            except Exception as acct_e:
-                failures += 1
-                logger.warning(f"email_auto_translate account scan failed for {getattr(acct, 'id', '?')}: {acct_e}")
-            finally:
-                if imap:
-                    try:
-                        imap.logout()
-                    except Exception:
-                        pass
-
-        if translated == 0 and same_language == 0:
-            result = (
-                f"no uncached foreign-language emails found "
-                f"(examined {examined}, cached {cached}, skipped {skipped}, failures {failures})"
-            )
-            if failures:
-                return f"Email Auto Translate failed: {result}", False
-            raise TaskNoop(result)
-        return (
-            f"Email Auto Translate cached {translated} translation(s), marked {same_language} same-language "
-            f"(examined {examined}, already cached {cached}, skipped {skipped}, failures {failures})",
-            True,
-        )
-    except TaskNoop:
-        raise
-    except Exception as e:
-        logger.error(f"email_auto_translate action failed: {e}")
-        return str(e), False
-
-
 _TYPE_COLORS = {
     "work":     "#5b8abf",  # blue
     "personal": "#a07ae0",  # purple
@@ -980,7 +693,6 @@ async def action_classify_events(owner: str, **kwargs) -> Tuple[str, bool]:
                     f"EVENTS: {_json.dumps(items)}"
                 )
                 try:
-                    await wait_for_interactive_quiet("calendar classification action")
                     raw = await llm_call_async_with_fallback(
                         llm_candidates,
                         messages=[{"role": "user", "content": prompt}],
@@ -1049,44 +761,19 @@ async def action_extract_email_events(owner: str, **kwargs) -> Tuple[str, bool]:
     import asyncio as _aio
     try:
         from routes.email_pollers import _run_auto_summarize_once
-        account_id = _email_task_account_id(kwargs)
+        try:
-        attempts = [
+            # Hard wall-clock budget: 5 min total. Per-LLM call already has its own timeout.
-            ("3d window, 3 emails", 3, 3, 240),
+            result = await _aio.wait_for(
-            ("3d window, 2 emails", 3, 2, 150),
+                _run_auto_summarize_once(
-            ("1d window, 1 email", 1, 1, 90),
+                    do_summary=False, do_reply=False, do_calendar=True, days_back=3,
-        ]
+                ),
-        timed_out = []
+                timeout=300,
-        last_result = ""
-        for label, days_back, max_process, timeout in attempts:
-            try:
-                result = await _aio.wait_for(
-                    _run_auto_summarize_once(
-                        do_summary=False,
-                        do_reply=False,
-                        do_calendar=True,
-                        days_back=days_back,
-                        account_id=account_id,
-                        max_process=max_process,
-                    ),
-                    timeout=timeout,
-                )
-                last_result = result or ""
-                if _result_is_config_error(result):
-                    return f"{result} ({label})", False
-                if _result_has_work(result):
-                    suffix = f"{label}" if not timed_out else f"{label}; retried after timeout"
-                    return f"{result} ({suffix})", True
-                raise TaskNoop(f"email→calendar: {result or 'no new emails'} ({label})")
-            except _aio.TimeoutError:
-                timed_out.append(label)
-                logger.warning(f"email calendar extraction timed out for {label}; retrying smaller batch")
-                continue
-        if timed_out:
-            raise TaskNoop(
-                "email→calendar: calendar extraction timed out on smaller batches; "
-                "will retry on the next scheduled run"
             )
-        raise TaskNoop(f"email→calendar: {last_result or 'no new emails'}")
+            if not _result_has_work(result):
+                raise TaskNoop(f"email→calendar: {result or 'no new emails'}")
+            return f"{result} (3d window)", True
+        except _aio.TimeoutError:
+            return "Email→calendar pass exceeded 5 min budget — try fewer emails or a faster model", False
     except Exception as e:
         logger.error(f"extract_email_events action failed: {e}")
         return str(e), False
@@ -1255,7 +942,6 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo
             )
 
             try:
-                await wait_for_interactive_quiet("sender signature action")
                 raw = await llm_call_async_with_fallback(
                     candidates,
                     messages=[{"role": "user", "content": prompt}],
@@ -1813,15 +1499,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
         STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
         AGE_CUTOFF = _dt.utcnow() - _td(days=7)
-        TRIAGE_VERSION = 10
+        TRIAGE_VERSION = 3
         CATEGORY_TAGS = {
-            "bills", "receipt", "travel", "calendar", "action-needed",
+            "newsletter", "marketing", "notification", "finance", "bills",
-        }
+            "receipt", "travel", "security", "shopping", "social", "work",
-        VISIBLE_EMAIL_TAGS = CATEGORY_TAGS | {"urgent", "reply-soon"}
+            "personal", "calendar",
-        MANAGED_TAGS = VISIBLE_EMAIL_TAGS | {
-            "newsletter", "marketing", "notification", "finance", "security",
-            "shopping", "social", "work", "personal", "legal", "support", "promo",
         }
+        MANAGED_TAGS = CATEGORY_TAGS | {"urgent", "reply-soon", "promo"}
 
         # ── 1. Resolve LLM candidates (utility primary + utility fallbacks; fall
         # through to default chat as a last resort).
@@ -1830,8 +1514,6 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
         if not candidates:
             return "No LLM endpoint available", False
 
-        target_account_id = _email_task_account_id(kwargs)
-
         # ── 2. Enumerate enabled accounts. Match this task's owner AND fall
         # back to the legacy "unowned account whose imap_user / from_address
         # == this owner" pattern — same rule `_get_email_config` uses, so a
@@ -1844,8 +1526,6 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                 unowned = _or(_EA.owner == None, _EA.owner == "")  # noqa: E711
                 same_mailbox = _or(_EA.imap_user == owner, _EA.from_address == owner)
                 q = q.filter(_or(_EA.owner == owner, _and(unowned, same_mailbox)))
-            if target_account_id:
-                q = q.filter(_EA.id == target_account_id)
             accounts = q.all()
         finally:
             db.close()
@@ -1854,95 +1534,12 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
 
         urgency_prompt = settings.get("urgent_email_prompt", "")
         per_uid_scores = {}   # key = "<acc_id>:<uid>" → {"score": 0-3, "reason": "..."}
-        all_unread_keys = set()
+        all_unread_keys = set()  # for cache pruning
         llm_attempts = 0
         saved_classifications = 0
         failed_classifications = []
-        tag_write_details = []
         scanned = 0
 
-        def _heuristic_email_verdict(item: dict) -> dict:
-            blob = (
-                f"{item.get('headers','')}\n{item.get('from','')}\n"
-                f"{item.get('subject','')}\n{item.get('body','')}"
-            ).lower()
-            response_tags = []
-            type_candidates = []
-
-            def add_response(tag: str):
-                if tag in CATEGORY_TAGS and tag not in response_tags:
-                    response_tags.append(tag)
-
-            def add_type(tag: str):
-                if tag in CATEGORY_TAGS and tag not in type_candidates:
-                    type_candidates.append(tag)
-
-            bulkish = bool(_re.search(
-                r"\b(list-unsubscribe|list-id|mailchimp|mailchimpapp|view this email in your browser|unsubscribe|newsletter|digest|precedence:\s*bulk)\b",
-                blob,
-            ))
-            marketingish = bool(_re.search(
-                r"\b(advertisement|sponsored|promo|promotion|sale|discount|offer|limited time|deal|coupon|shop now|buy now|membership|rewards?)\b",
-                blob,
-            ))
-            if bulkish or marketingish:
-                add_type("newsletter")
-            if _re.search(r"\b(receipt|order|注文|payment confirmation|delivery|shipment|tracking|お届け|購入)\b", blob):
-                add_type("receipt")
-            if _re.search(r"\b(bill|billing|amount due|overdue|pay by|payment due|subscription could not be renewed)\b", blob):
-                add_type("bills")
-            if _re.search(r"\b(court|charge|legal|lawyer|solicitor|claim|judgment|registration fee|debt)\b", blob):
-                add_type("legal")
-            if _re.search(r"\b(flight|hotel|booking|reservation|itinerary|train|ticket|trip|旅|予約)\b", blob):
-                add_type("travel")
-            if _re.search(r"\b(ticket|case|support|helpdesk|request)\b", blob):
-                add_type("support")
-            if _re.search(r"\b(meeting|appointment|calendar|invite|event|schedule|予定|保育園|連絡帳)\b", blob):
-                add_response("calendar")
-            if _re.search(
-                r"\b(action required|required action|please reply|please respond|deadline|by \d{1,2} |pay within|submit|sign|confirm|approval|waiting outside|locked out|can't get in|cannot get in|invoice|bill|billing|payment|balance|debt|subscription|renewal|overdue|amount due|court|charge|legal|lawyer|solicitor|claim|judgment)\b",
-                blob,
-            ):
-                add_response("action-needed")
-
-            type_priority = ("bills", "receipt", "travel")
-            tags = [*response_tags]
-            for type_tag in type_priority:
-                if type_tag in type_candidates and type_tag not in tags:
-                    tags.append(type_tag)
-                if len(tags) >= len(response_tags) + 2:
-                    break
-
-            score = 0
-            reason = "categorized by email metadata"
-            if "action-needed" in response_tags:
-                score = 2
-                reason = "action likely needed"
-            if _re.search(r"\b(urgent|immediately|final notice|locked out|waiting outside|can't get in|cannot get in)\b", blob):
-                score = 3
-                reason = "urgent wording"
-            if (bulkish or marketingish) and score < 2:
-                score = 0
-                reason = "bulk marketing/newsletter"
-
-            _from_raw = item.get("from", "") or ""
-            if "<" in _from_raw:
-                _from_short = _from_raw.split("<", 1)[0].strip().strip('"') or _from_raw
-            else:
-                _from_short = _from_raw
-            return {
-                "score": max(0, min(3, score)),
-                "tags": tags[:4],
-                "spam": False,
-                "reason": reason,
-                "subject": (item.get("subject") or "")[:200],
-                "from": _from_short[:120],
-                "triage_version": TRIAGE_VERSION,
-                "message_id": (item.get("message_id") or "").strip(),
-                "unread": bool(item.get("unread")),
-                "ts": _time.time(),
-            }
-
         # ── 3. Per-account scan: pull headers + lightweight body for new UIDs
         # since 7 days ago, score via LLM, cache the verdict.
         for acc in accounts:
@@ -1958,13 +1555,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                 conn = _imap_connect(account.id)
                 try:
                     conn.select("INBOX", readonly=True)
-                    # Tag recent inbox mail, not only unread mail. Urgency
+                    # IMAP date is the only practical pre-filter — UNSEEN AND
-                    # reminders below still only notify for unread messages.
+                    # SINCE 7-days-ago. Date format is DD-Mon-YYYY.
                     since_str = AGE_CUTOFF.strftime("%d-%b-%Y")
-                    status, data = conn.uid("SEARCH", None, f'(SINCE {since_str})')
+                    status, data = conn.search(None, f'(UNSEEN SINCE {since_str})')
                     if status != "OK" or not data or not data[0]:
                         return results
-                    uids = data[0].split()[-30:]
+                    uids = data[0].split()
                     for uid_b in uids:
                         uid = uid_b.decode() if isinstance(uid_b, bytes) else str(uid_b)
                         key = f"{account.id}:{uid}"
@@ -1976,14 +1573,9 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                             continue
                         # Pull headers + first ~800 chars of plaintext body.
                         try:
-                            st, msg_data = conn.uid("FETCH", uid_b, "(UID FLAGS RFC822.HEADER BODY.PEEK[TEXT]<0.800>)")
+                            st, msg_data = conn.fetch(uid_b, "(RFC822.HEADER BODY.PEEK[TEXT]<0.800>)")
                             if st != "OK" or not msg_data:
                                 continue
-                            flags_blob = b" ".join(
-                                part[0] for part in msg_data
-                                if isinstance(part, tuple) and part and isinstance(part[0], (bytes, bytearray))
-                            )
-                            is_unread = b"\\Seen" not in flags_blob
                             # Headers + body land in different tuples in the
                             # response — concatenate the bytes for parsing.
                             raw = b""
@@ -2043,7 +1635,6 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                                 "headers": header_blob,
                                 "body": body_snippet.strip(),
                                 "message_id": (msg.get("Message-ID") or "").strip(),
-                                "unread": is_unread,
                             })
                         except Exception as _fe:
                             logger.debug(f"urgency: header fetch for uid {uid} failed: {_fe}")
@@ -2061,33 +1652,25 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
             for item in items:
                 scanned += 1
                 key = item["key"]
-                if item.get("unread"):
+                all_unread_keys.add(key)
-                    all_unread_keys.add(key)
                 if item.get("cached"):
-                    cached_v = dict(item["cached"])
+                    per_uid_scores[key] = item["cached"]
-                    cached_v["unread"] = bool(item.get("unread"))
-                    per_uid_scores[key] = cached_v
                     continue
                 # Skip uids we couldn't fetch (no subject/from/body).
                 if not item.get("subject") and not item.get("from"):
                     continue
-                verdict = _heuristic_email_verdict(item)
-                cache.setdefault("uids", {})[item["uid"]] = verdict
-                per_uid_scores[key] = verdict
-                saved_classifications += 1
-                continue
                 # ── LLM-classify. JSON-only response; bullet-proof parse.
                 llm_attempts += 1
                 prompt = (
-                    "You are triaging ONE email. Return ONLY JSON: "
+                    "You are triaging ONE unread email. Return ONLY JSON: "
                     "{\"score\":0|1|2|3,\"tags\":[\"...\"],\"spam\":false,"
                     "\"reason\":\"one short phrase\"}.\n"
                     "0 = trivial / promotional · 1 = informational, no reply needed · "
                     "2 = should reply within a day · 3 = urgent, reply now (deadline, blocker).\n\n"
-                    "Allowed visible tags: urgent, reply-soon, action-needed, calendar, bills, receipt, travel.\n"
+                    "Allowed tags: newsletter, marketing, notification, finance, bills, receipt, "
-                    "Use action-needed when the user likely needs to reply, pay, sign, book, or decide. "
+                    "travel, security, shopping, social, work, personal, calendar.\n"
-                    "Use bills for bills or debts, receipt for purchases/deliveries, travel for reservations/trips, "
+                    "Use marketing for ads, promos, sales, offers, and cold sales. Use newsletter "
-                    "and calendar only when a calendar event/reminder is involved. spam=true for scams, phishing, "
+                    "for newsletters, digests, and recurring content. spam=true for scams, phishing, "
                     "junk, cold sales, generic ads, or no-personal-action bulk mail.\n"
                     "Important: 'I'm outside', 'I am outside', 'waiting outside', 'at the door', "
                     "'locked out', or 'can't get in' means score 3 unless clearly historical.\n\n"
@@ -2096,7 +1679,6 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                     f"Snippet:\n{item.get('body','')}\n"
                 )
                 try:
-                    await wait_for_interactive_quiet("email urgency action")
                     raw = await llm_call_async_with_fallback(
                         candidates,
                         [{"role": "user", "content": prompt}],
@@ -2157,10 +1739,14 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         r"\b(advertisement|sponsored|promo|promotion|sale|discount|offer|limited time|deal|tickets?|tour|merch|stream|purchase|sold out|low tickets|coupon|shop now|buy now)\b",
                         _blob,
                     ))
+                    if "newsletter" not in tags and bulkish:
+                        tags.append("newsletter")
+                    if "marketing" not in tags and marketingish:
+                        tags.append("marketing")
                     if (bulkish or marketingish) and score < 2:
                         score = 0
                         if not reason or "urgent" in reason.lower():
-                            reason = "bulk mail; no personal reply needed"
+                            reason = "Bulk marketing/newsletter; no personal reply needed"
                     # Strip "Name <addr>" to bare display name for compact summary.
                     _from_raw = item.get("from", "") or ""
                     if "<" in _from_raw:
@@ -2178,7 +1764,6 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         # Cache the message_id too so re-scans of already-cached
                         # UIDs can still write the inbox tag without re-LLM'ing.
                         "message_id": (item.get("message_id") or "").strip(),
-                        "unread": bool(item.get("unread")),
                         "ts": _time.time(),
                     }
                     cache.setdefault("uids", {})[item["uid"]] = verdict
@@ -2193,9 +1778,9 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                     logger.debug(f"urgency: LLM classify failed for {key}: {e}")
                     continue
 
-            # ── Prune cache entries for UIDs that are no longer in the recent
+            # ── Prune cache entries for UIDs that are no longer unread (replied
-            # scan window. Read messages remain cached because tags are useful
+            # / archived / deleted). Compare against `items` (everything UNSEEN
-            # on read mail too; unread state is refreshed per scan above.
+            # in this scan window).
             seen_uids = {it["uid"] for it in items}
             cache_uids = cache.get("uids", {})
             for stale in [u for u in cache_uids if u not in seen_uids]:
@@ -2230,17 +1815,15 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         _tag = str(_tag).strip().lower().replace("_", "-")
                         if _tag == "promo":
                             _tag = "marketing"
-                        if _tag == "action-needed" and any(t in _new_tags for t in ("urgent", "reply-soon")):
+                        if _tag in CATEGORY_TAGS and _tag not in _new_tags:
-                            continue
-                        if _tag in VISIBLE_EMAIL_TAGS and _tag not in _new_tags:
                             _new_tags.append(_tag)
                     _spam = 1 if _v.get("spam") else 0
                     # _key is "<account_id>:<uid>" — extract uid for the row.
-                    _acc_id, _uid_only = (_key.split(":", 1) + [""])[:2]
+                    _uid_only = _key.split(":", 1)[-1]
                     _owner_key = owner or ""
                     _row = _conn.execute(
-                        "SELECT tags FROM email_tags WHERE message_id=? AND owner=? AND account_id=?",
+                        "SELECT tags FROM email_tags WHERE message_id=? AND owner=?",
-                        (_msg_id, _owner_key, _acc_id),
+                        (_msg_id, _owner_key),
                     ).fetchone()
                     if _row:
                         try:
@@ -2259,42 +1842,23 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
                         for _tag in _new_tags:
                             if _tag not in _existing:
                                 _existing.append(_tag)
-                        if _new_tags or _spam:
-                            tag_write_details.append({
-                                "uid": _uid_only,
-                                "subject": _v.get("subject", ""),
-                                "from": _v.get("from", ""),
-                                "tags": list(_new_tags),
-                                "spam": _spam,
-                                "reason": _v.get("reason", ""),
-                                "updated": True,
-                            })
                         _conn.execute(
                             "UPDATE email_tags SET tags=?, spam_verdict=?, spam_reason=?, uid=?, folder=?, subject=?, sender=? "
-                            "WHERE message_id=? AND owner=? AND account_id=?",
+                            "WHERE message_id=? AND owner=?",
                             (_json.dumps(_existing), _spam, _v.get("reason", ""), _uid_only, "INBOX",
-                             _v.get("subject", ""), _v.get("from", ""), _msg_id, _owner_key, _acc_id),
+                             _v.get("subject", ""), _v.get("from", ""), _msg_id, _owner_key),
                         )
                     else:
                         if not _new_tags and not _spam:
                             continue
                         _conn.execute(
                             "INSERT INTO email_tags "
-                            "(message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict, spam_reason, created_at) "
+                            "(message_id, owner, uid, folder, subject, sender, tags, spam_verdict, spam_reason, created_at) "
-                            "VALUES (?, ?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?)",
+                            "VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?)",
-                            (_msg_id, _owner_key, _acc_id, _uid_only, _v.get("subject", ""),
+                            (_msg_id, _owner_key, _uid_only, _v.get("subject", ""),
                              _v.get("from", ""), _json.dumps(_new_tags), _spam, _v.get("reason", ""),
                              _dt2.utcnow().isoformat()),
                         )
-                        tag_write_details.append({
-                            "uid": _uid_only,
-                            "subject": _v.get("subject", ""),
-                            "from": _v.get("from", ""),
-                            "tags": list(_new_tags),
-                            "spam": _spam,
-                            "reason": _v.get("reason", ""),
-                            "updated": False,
-                        })
                 _conn.commit()
             finally:
                 _conn.close()
@@ -2302,7 +1866,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
             logger.warning(f"urgency: bulk tag write failed: {_te}")
 
         # ── 4. Aggregate state. urgent = score ≥ 2.
-        urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2 and v.get("unread")]
+        urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2]
         max_score = max((v.get("score", 0) for v in per_uid_scores.values()), default=0)
         total_urgent = len(urgent_keys)
 
@@ -2411,28 +1975,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
             f"reply-soon {tier_counts[2]} · info {tier_counts[1]} · trivial {tier_counts[0]} · "
             f"{saved_classifications} saved classifications"
         )
-        if failed_classifications:
+        if llm_attempts != saved_classifications:
-            head += f" · {len(failed_classifications)} failed"
+            head += f" · {llm_attempts - saved_classifications} failed"
         if newly_notified:
             head += f" · notified {len(newly_notified)}"
         if notify_failed:
             head += f" · notify failed {len(notify_failed)}"
 
-        def _fmt_tag_write(v):
-            subj = (v.get("subject") or "(no subject)")[:80]
-            frm = v.get("from") or ""
-            tags = list(v.get("tags") or [])
-            if v.get("spam"):
-                tags.append("spam")
-            tag_txt = ", ".join(tags) if tags else "cleared managed tags"
-            why = v.get("reason") or ""
-            op = "updated" if v.get("updated") else "created"
-            line = f"- **{subj}**" + (f" — _{frm}_" if frm else "")
-            line += f" — `{tag_txt}` ({op})"
-            if why:
-                line += f" · {why}"
-            return line
-
         def _fmt_one(v, newly_notified_set, failed_set, key):
             subj = (v.get("subject") or "(no subject)")[:80]
             frm = v.get("from") or ""
@@ -2448,13 +1997,6 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
         for k, v in per_uid_scores.items():
             by_tier.setdefault(v.get("score", 0), []).append((k, v))
         lines = [head]
-        if tag_write_details:
-            lines.append("")
-            lines.append(f"**Applied tags ({len(tag_write_details)}):**")
-            for v in tag_write_details[:16]:
-                lines.append(_fmt_tag_write(v))
-            if len(tag_write_details) > 16:
-                lines.append(f"…and {len(tag_write_details) - 16} more")
         tier_labels = {3: "Urgent", 2: "Reply soon", 1: "Informational", 0: "Trivial"}
         for tier in (3, 2, 1, 0):
             items_t = by_tier.get(tier, [])
@@ -2526,7 +2068,6 @@ async def action_cookbook_serve(
         end_after_min = int(cfg.get("end_after_min") or 0)
     except Exception:
         end_after_min = 0
-    set_default = bool(cfg.get("set_default", True))
 
     state_path = Path(COOKBOOK_STATE_FILE)
     try:
@@ -2613,51 +2154,6 @@ async def action_cookbook_serve(
         return f"Launch rejected: {data.get('error') or data.get('detail') or 'unknown'}", False
 
     sid = data.get("session_id") or ""
-    endpoint_id = data.get("endpoint_id") or ""
-    # Scheduled serves are usually meant to become the active local model for
-    # chat/tools while their time window is open. Persist both endpoint and
-    # model so task/utility/default resolution does not keep routing to a stale
-    # API fallback. Allow explicit opt-out with {"set_default": false}.
-    if endpoint_id and set_default:
-        try:
-            selected_model = repo_id
-            try:
-                from core.database import SessionLocal as _SL, ModelEndpoint as _ME
-                _db = _SL()
-                try:
-                    _ep = _db.query(_ME).filter(_ME.id == endpoint_id).first()
-                    if _ep and _ep.cached_models:
-                        _models = json.loads(_ep.cached_models or "[]")
-                        if isinstance(_models, list) and _models:
-                            selected_model = str(_models[0])
-                finally:
-                    _db.close()
-            except Exception:
-                pass
-            from src.settings import load_settings as _load_settings, save_settings as _save_settings
-            _settings = _load_settings()
-            _settings["default_endpoint_id"] = endpoint_id
-            _settings["default_model"] = selected_model
-            # Keep background tasks aligned unless the user explicitly chose a
-            # separate task model.
-            if not (_settings.get("task_endpoint_id") or "").strip():
-                _settings["task_endpoint_id"] = endpoint_id
-                _settings["task_model"] = selected_model
-            if not (_settings.get("utility_endpoint_id") or "").strip():
-                _settings["utility_endpoint_id"] = endpoint_id
-                _settings["utility_model"] = selected_model
-            _save_settings(_settings)
-            if owner:
-                from routes.prefs_routes import _load_for_user, _save_for_user
-                _prefs = _load_for_user(owner)
-                _prefs["default_endpoint_id"] = endpoint_id
-                _prefs["default_model"] = selected_model
-                if not (_prefs.get("utility_endpoint_id") or "").strip():
-                    _prefs["utility_endpoint_id"] = endpoint_id
-                    _prefs["utility_model"] = selected_model
-                _save_for_user(owner, _prefs)
-        except Exception as e:
-            logger.warning(f"cookbook_serve: default endpoint update failed: {e}")
     # Register the new task in cookbook_state.json + stamp it with our
     # scheduler-owner markers. /api/model/serve spawns the tmux session
     # but leaves the state-write to the UI — when a scheduled action
@@ -2701,16 +2197,12 @@ async def action_cookbook_serve(
                     "sshPort": ssh_port or "",
                     "platform": platform or "linux",
                     "_serveReady": False,
-                    "_endpointAdded": bool(endpoint_id),
+                    "_endpointAdded": False,
                 }
                 tasks.append(existing)
             # Stamp ownership + end-at on the task entry.
             existing["_scheduledByTask"] = task_name or ""
             existing["_scheduledByOwner"] = owner or ""
-            if endpoint_id:
-                existing["_endpointId"] = endpoint_id
-                existing["endpointId"] = endpoint_id
-                existing["_endpointAdded"] = True
             if end_after_min > 0:
                 existing["_scheduledStopAtMs"] = int(_time.time() * 1000) + end_after_min * 60 * 1000
             fresh["tasks"] = tasks
@@ -2738,7 +2230,6 @@ BUILTIN_ACTIONS = {
     "tidy_research": action_tidy_research,
     "summarize_emails": action_summarize_emails,
     "draft_email_replies": action_draft_email_replies,
-    "email_auto_translate": action_email_auto_translate,
     "extract_email_events": action_extract_email_events,
     "classify_events": action_classify_events,
     # ping_events removed from the user-facing registry. Calendar reminders
@@ -2763,7 +2254,6 @@ BUILTIN_ACTION_INFO = {
     "tidy_research": "Remove orphaned research files (sessions that were deleted)",
     "summarize_emails": "Pre-generate AI summaries for new inbox emails",
     "draft_email_replies": "Pre-draft AI reply suggestions for new inbox emails",
-    "email_auto_translate": "Detect foreign-language emails and cache translated text for the email reader",
     "extract_email_events": "Scan emails for booking/meeting confirmations and auto-add to calendar",
     "classify_events": "Tag upcoming events with importance (low/normal/high/critical) and type (work/health/travel/etc.); colors them too",
     "daily_brief": "Build a morning digest: today's calendar, unread email count + top senders, active todos",

src/caldav_sync.py

@@ -25,7 +25,6 @@ Design notes:
 import asyncio
 import hashlib
 import ipaddress
-import json
 import logging
 import os
 import socket
@@ -501,7 +500,6 @@ def _event_payload(ev) -> dict:
         "all_day": ev.all_day,
         "is_utc": ev.is_utc,
         "rrule": ev.rrule or "",
-        "recurrence_exdates": json.loads(ev.recurrence_exdates or "[]") if getattr(ev, "recurrence_exdates", "") else [],
     }
 
 

src/caldav_writeback.py

@@ -33,8 +33,7 @@ def build_event_ical(ev: dict) -> str:
     """Serialize a local event dict to a VCALENDAR/VEVENT iCalendar string.
 
     ``ev`` keys: uid, summary, description, location, dtstart (datetime),
-    dtend (datetime), all_day (bool), is_utc (bool), rrule (str),
+    dtend (datetime), all_day (bool), is_utc (bool), rrule (str).
-    recurrence_exdates (list[str]).
     Mirrors how the pull path interprets is_utc/all_day so a round-trip is stable.
     """
     from icalendar import Calendar, Event as iEvent
@@ -71,15 +70,6 @@ def build_event_ical(ev: dict) -> str:
             ve.add("rrule", vRecur.from_ical(ev["rrule"]))
         except Exception:
             logger.debug("CalDAV write-back: skipping unparseable rrule %r", ev.get("rrule"))
-    for exdate in ev.get("recurrence_exdates") or []:
-        try:
-            if ev.get("all_day"):
-                ve.add("exdate", datetime.strptime(exdate[:10], "%Y-%m-%d").date())
-            else:
-                dt = datetime.strptime(exdate[:16], "%Y-%m-%dT%H:%M")
-                ve.add("exdate", dt.replace(tzinfo=timezone.utc) if ev.get("is_utc") else dt)
-        except Exception:
-            logger.debug("CalDAV write-back: skipping unparseable exdate %r", exdate)
 
     cal.add_component(ve)
     return cal.to_ical().decode("utf-8")

src/chat_handler.py

@@ -29,34 +29,6 @@ from src.youtube_handler import (
 logger = logging.getLogger(__name__)
 
 
-def _sync_upload_vision_to_gallery(file_info: Dict[str, Any], owner: Optional[str], text: str) -> None:
-    file_hash = (file_info or {}).get("hash")
-    if not file_hash or not text:
-        return
-    try:
-        from core.database import GalleryImage, SessionLocal
-        db = SessionLocal()
-        try:
-            q = db.query(GalleryImage).filter(
-                GalleryImage.file_hash == file_hash,
-                GalleryImage.is_active == True,  # noqa: E712
-            )
-            if owner:
-                q = q.filter(GalleryImage.owner == owner)
-            img = q.first()
-            if not img:
-                return
-            img.caption = text.strip()
-            db.commit()
-        except Exception:
-            db.rollback()
-            raise
-        finally:
-            db.close()
-    except Exception as e:
-        logger.warning("Failed to sync upload vision text to gallery: %s", e)
-
-
 class ChatHandler:
     """Handles chat operations for both streaming and non-streaming endpoints."""
 
@@ -235,7 +207,6 @@ class ChatHandler:
                                     _vtext = _vf.read().strip()
                                 if _vtext:
                                     enhanced_message += f"\n[User-corrected caption / OCR for this image — treat as authoritative]:\n{_vtext}"
-                                    _sync_upload_vision_to_gallery(file_info, owner, _vtext)
                                     _m = meta_by_id.get(att_id)
                                     if _m is not None:
                                         _m["vision"] = _vtext
@@ -255,7 +226,6 @@ class ChatHandler:
                                     cached_desc = _vf.read().strip()
                                 if cached_desc and not cached_desc.startswith("["):
                                     vl_desc = cached_desc
-                                    _sync_upload_vision_to_gallery(file_info, owner, vl_desc)
                             except Exception:
                                 vl_desc = None
                         if not vl_desc:
@@ -267,7 +237,6 @@ class ChatHandler:
                                     os.makedirs(os.path.join(UPLOAD_DIR, ".vision"), exist_ok=True)
                                     with open(_vcache, "w", encoding="utf-8") as _vf:
                                         _vf.write(vl_desc)
-                                    _sync_upload_vision_to_gallery(file_info, owner, vl_desc)
                                 except Exception:
                                     pass
                         enhanced_message = f"{enhanced_message}\n\n[Image: {file_info['name']}]\n{vl_desc}"

src/context_budget.py

@@ -18,13 +18,6 @@ DEFAULT_BUDGET = 6000
 DEFAULT_HEADROOM = 0.85
 
 
-def _int_or_zero(value) -> int:
-    try:
-        return int(value or 0)
-    except (TypeError, ValueError):
-        return 0
-
-
 def compute_input_token_budget(
     configured: int,
     context_length: int,
@@ -55,8 +48,8 @@ def compute_input_token_budget(
         - When the window is unknown (context_length <= 0), use the conservative
           ``default`` budget and do NOT scale off the fallback.
     """
-    configured = _int_or_zero(configured)
+    configured = int(configured or 0)
-    context_length = _int_or_zero(context_length)
+    context_length = int(context_length or 0)
 
     if explicit and configured > 0:
         return min(configured, context_length) if context_length > 0 else configured

src/cookbook_serve_lifecycle.py

@@ -37,13 +37,6 @@ async def _delete_endpoint_for_task(task: dict) -> None:
     the picker (probe goes offline; chats still try to route there) and
     the user has to delete it by hand in Settings -> Endpoints.
     """
-    endpoint_id = (task.get("_endpointId") or task.get("endpointId") or "").strip()
-    if not endpoint_id:
-        logger.info(
-            "cookbook_serve_lifecycle: task %s has no endpoint id; skipping endpoint deletion",
-            task.get("sessionId") or task.get("id") or "",
-        )
-        return
     import re as _re
     payload = task.get("payload") or {}
     cmd = str(payload.get("_cmd") or "")
@@ -73,10 +66,13 @@ async def _delete_endpoint_for_task(task: dict) -> None:
             if r.status_code >= 400:
                 return
             eps = r.json() if r.content else []
-            # Delete only the endpoint created by this scheduled serve. URL
+            # Prefer exact URL match; fall back to host:port substring so we
-            # matching is unsafe because a later scheduled serve can reuse the
+            # still catch the case where 0.0.0.0 vs the registered host
-            # same host:port after an older task has gone stale.
+            # representation diverged.
-            ep = next((e for e in eps if e.get("id") == endpoint_id), None)
+            ep = next((e for e in eps if e.get("base_url") == base_url), None)
+            if not ep:
+                hostport = f"{host}:{port}"
+                ep = next((e for e in eps if hostport in (e.get("base_url") or "")), None)
             if ep:
                 await client.delete(
                     f"{internal_api_base()}/api/model-endpoints/{ep['id']}",

src/copilot.py

@@ -230,10 +230,7 @@ def request_flags(messages) -> tuple:
     """
     msgs = messages or []
     last = msgs[-1] if msgs else None
-    # A message element can be a non-dict (clients send `"messages": ["hi"]`);
+    agent = bool(last) and last.get("role") != "user"
-    # the vision loop below already guards each element with isinstance, so do
-    # the same here rather than call .get() on a bare string.
-    agent = isinstance(last, dict) and last.get("role") != "user"
     vision = False
     for m in msgs:
         content = m.get("content") if isinstance(m, dict) else None

src/document_actions.py

@@ -6,7 +6,7 @@ Reusable document actions callable from both REST routes and the task scheduler.
 
 import logging
 import re
-from datetime import datetime, timezone
+from datetime import datetime
 
 logger = logging.getLogger(__name__)
 
@@ -77,28 +77,10 @@ async def run_document_tidy(owner: str) -> str:
         deleted = 0
         kept = 0
         survivors = []  # docs that pass the junk rules, considered for dedup
-        now = datetime.now(timezone.utc)
 
         for doc in docs:
-            created = doc.created_at
-            if created and created.tzinfo is None:
-                created = created.replace(tzinfo=timezone.utc)
-
-            # Skip freshly created documents to avoid deleting them while the user is actively editing
-            if created and (now - created).total_seconds() < 900:  # 15 minutes
-                survivors.append(doc)
-                continue
-
             content = (doc.current_content or "").strip()
             title = (doc.title or "").strip().lower()
-            is_fresh_empty = (
-                not content
-                and created is not None
-                and (now - created).total_seconds() < 1800
-            )
-            if is_fresh_empty:
-                survivors.append(doc)
-                continue
 
             # Strip markdown noise to get "real" character count
             stripped = re.sub(r"^#{1,6}\s+", "", content, flags=re.MULTILINE)  # headers

src/document_processor.py

@@ -440,10 +440,7 @@ def build_user_content(
             try:
                 with open(path, "rb") as image_file:
                     encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
-                # Extensionless uploads (e.g. a pasted screenshot) have no ext,
+                image_format = ext[1:]
-                # so fall back to the resolved MIME subtype rather than emitting
-                # an invalid "data:image/;base64," with an empty subtype.
-                image_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("image/") else "png")
                 content.append({
                     "type": "image_url",
                     "image_url": {"url": f"data:image/{image_format};base64,{encoded_string}"},
@@ -459,7 +456,7 @@ def build_user_content(
             try:
                 with open(path, "rb") as audio_file:
                     encoded_string = base64.b64encode(audio_file.read()).decode("utf-8")
-                audio_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("audio/") else "mpeg")
+                audio_format = ext[1:]
                 content.append({
                     "type": "audio",
                     "audio": {"url": f"data:audio/{audio_format};base64,{encoded_string}"},

src/endpoint_resolver.py

@@ -47,33 +47,6 @@ def _endpoint_cached_models(ep) -> list:
     return models if isinstance(models, list) else []
 
 
-def _endpoint_pinned_models(ep) -> list:
-    raw = getattr(ep, "pinned_models", None)
-    if not raw:
-        return []
-    try:
-        models = json.loads(raw) if isinstance(raw, str) else raw
-    except Exception:
-        return []
-    return models if isinstance(models, list) else []
-
-
-def _is_mlx_deepseek_v4_repo_id(model_id: str) -> bool:
-    return "mlx-community/deepseek-v4" in str(model_id or "").lower()
-
-
-def _is_mlx_deepseek_v4_shim_id(model_id: str) -> bool:
-    return "/.cache/odysseus/mlx-shims/deepseek-v4" in str(model_id or "").lower()
-
-
-def _filter_mlx_deepseek_v4_repo_when_shimmed(model_ids) -> list:
-    ids = list(model_ids or [])
-    has_shim = any(_is_mlx_deepseek_v4_shim_id(m) for m in ids)
-    if not has_shim:
-        return ids
-    return [m for m in ids if not _is_mlx_deepseek_v4_repo_id(m)]
-
-
 def _endpoint_hidden_models(ep) -> set:
     """Model ids the admin disabled on this endpoint (the UI's hidden list)."""
     raw = getattr(ep, "hidden_models", None)
@@ -94,15 +67,7 @@ def _endpoint_enabled_models(ep) -> list:
     raw first one resolves to a model that 400s ("requires terms acceptance").
     """
     hidden = _endpoint_hidden_models(ep)
-    merged = []
+    return [m for m in _endpoint_cached_models(ep) if m not in hidden]
-    seen = set()
-    for m in [*_endpoint_cached_models(ep), *_endpoint_pinned_models(ep)]:
-        if not isinstance(m, str) or not m or m in seen:
-            continue
-        seen.add(m)
-        merged.append(m)
-    merged = _filter_mlx_deepseek_v4_repo_when_shimmed(merged)
-    return [m for m in merged if m not in hidden]
 
 
 def resolve_endpoint_runtime(ep, owner: Optional[str] = None) -> Tuple[str, Optional[str]]:

src/host_docker_access.py

@@ -1,62 +0,0 @@
-"""Policy checks for explicit host Docker access from a container."""
-
-import os
-import stat
-from collections.abc import Mapping
-
-
-HOST_DOCKER_ENV_VAR = "ODYSSEUS_ENABLE_HOST_DOCKER"
-HOST_DOCKER_SOCKET_PATH = "/var/run/docker.sock"
-
-HOST_DOCKER_ACCESS_HINT = (
-    "Local Docker daemon access is disabled inside the Odysseus container; a "
-    "Docker CLI alone is not enough. Default Docker Compose intentionally does "
-    "not mount the host Docker socket. Raw socket access is high-trust and can "
-    "grant broad control over the host Docker daemon. If you accept that risk, "
-    "enable docker/host-docker.yml. Remote server Docker workflows over SSH "
-    "remain preferred."
-)
-
-
-def running_in_container(
-    dockerenv_path: str = "/.dockerenv",
-    cgroup_path: str = "/proc/1/cgroup",
-) -> bool:
-    if os.path.exists(dockerenv_path):
-        return True
-    try:
-        with open(cgroup_path, "r", encoding="utf-8") as handle:
-            contents = handle.read()
-    except OSError:
-        return False
-    return any(token in contents for token in ("docker", "containerd", "kubepods"))
-
-
-def host_docker_access_enabled(
-    socket_path: str = HOST_DOCKER_SOCKET_PATH,
-    *,
-    environ: Mapping[str, str] | None = None,
-) -> bool:
-    env = os.environ if environ is None else environ
-    if env.get(HOST_DOCKER_ENV_VAR, "").strip().lower() != "true":
-        return False
-    try:
-        mode = os.stat(socket_path).st_mode
-    except OSError:
-        return False
-    return stat.S_ISSOCK(mode)
-
-
-def local_docker_available(
-    *,
-    cli_available: bool,
-    in_container: bool | None = None,
-    environ: Mapping[str, str] | None = None,
-    socket_path: str = HOST_DOCKER_SOCKET_PATH,
-) -> bool:
-    if not cli_available:
-        return False
-    containerized = running_in_container() if in_container is None else in_container
-    if not containerized:
-        return True
-    return host_docker_access_enabled(socket_path, environ=environ)

src/integrations.py

@@ -216,14 +216,7 @@ def _normalize_integration_base_url(base_url: Any) -> str:
 
 
 def _join_integration_url(base_url: str, path: str) -> str:
-    base = base_url.rstrip("/")
+    return urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
-    rel = path.lstrip("/")
-    if not rel:
-        # A bare "/" must resolve to the base URL itself, not base + "/".
-        # POST-to-base integrations (e.g. Discord webhooks) 404 on the
-        # trailing-slash variant of their URL.
-        return base
-    return urljoin(base + "/", rel)
 
 
 def load_integrations() -> List[Dict[str, Any]]:
@@ -401,22 +394,6 @@ async def execute_api_call(
         return {"error": "Path must not contain a fragment", "exit_code": 1}
 
     url = _join_integration_url(base_url, path)
-
-    # SSRF guard — same check used by the gallery endpoint, embeddings,
-    # CardDAV, and the reminder webhook sender. Link-local / metadata
-    # addresses (169.254.x.x — the cloud credential-exfil vector) are always
-    # rejected; INTEGRATION_API_BLOCK_PRIVATE_IPS=true also blocks RFC-1918 /
-    # loopback for locked-down deployments. Private stays allowed by default
-    # because LAN integrations (Home Assistant, Miniflux, ntfy) are the
-    # primary use case.
-    from src.url_safety import check_outbound_url
-    block_private = os.getenv(
-        "INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
-    ).lower() == "true"
-    ok, reason = check_outbound_url(url, block_private=block_private)
-    if not ok:
-        return {"error": f"URL rejected: {reason}", "exit_code": 1}
-
     method = method.upper()
 
     # Build headers

src/interactive_gate.py

@@ -1,203 +0,0 @@
-"""Foreground activity gate for background work.
-
-Background tasks are allowed to run only after normal UI/API traffic has
-settled. This keeps scheduled jobs and email pollers from competing with the
-user opening Odysseus, Cookbook, email, documents, notes, or other panels.
-"""
-
-from __future__ import annotations
-
-import asyncio
-from contextlib import asynccontextmanager
-import os
-import time
-
-
-_ACTIVE_REQUESTS = 0
-_LAST_ACTIVITY = 0.0
-_LAST_BROWSER_ACTIVITY = 0.0
-_COND: asyncio.Condition | None = None
-_COND_LOOP: asyncio.AbstractEventLoop | None = None
-
-
-def _enabled() -> bool:
-    return os.getenv("BACKGROUND_TASK_FOREGROUND_GATE", "true").lower() not in {"0", "false", "no", "off"}
-
-
-def _quiet_seconds() -> float:
-    try:
-        return max(0.0, float(os.getenv("BACKGROUND_TASK_QUIET_MS", "1500")) / 1000.0)
-    except Exception:
-        return 1.5
-
-
-def _max_wait_seconds() -> float:
-    """0 means wait indefinitely until the UI is quiet."""
-    try:
-        return max(0.0, float(os.getenv("BACKGROUND_TASK_MAX_WAIT_SECONDS", "0")))
-    except Exception:
-        return 0.0
-
-
-def _browser_active_seconds() -> float:
-    """How long a visible Odysseus browser heartbeat blocks background tasks."""
-    try:
-        return max(0.0, float(os.getenv("BACKGROUND_TASK_BROWSER_ACTIVE_SECONDS", "45")))
-    except Exception:
-        return 45.0
-
-
-def _condition() -> asyncio.Condition:
-    global _COND, _COND_LOOP
-    try:
-        loop = asyncio.get_running_loop()
-    except RuntimeError:
-        loop = None
-    if _COND is None or _COND_LOOP is not loop:
-        _COND = asyncio.Condition()
-        _COND_LOOP = loop
-    return _COND
-
-
-_PASSIVE_EXACT_PATHS = {
-    "/api/activity/heartbeat",
-    "/api/client-perf",
-    "/api/tasks/notifications",
-    "/api/research/active",
-    "/api/email/urgency-state",
-}
-
-_PASSIVE_PREFIXES = (
-    "/api/chat/stream_status",
-    "/api/health",
-    "/api/prefs",
-)
-
-
-def should_track_interactive_request(path: str, method: str = "GET") -> bool:
-    if not _enabled():
-        return False
-    if (method or "").upper() == "OPTIONS":
-        return False
-    if path in _PASSIVE_EXACT_PATHS:
-        return False
-    if any(path.startswith(prefix) for prefix in _PASSIVE_PREFIXES):
-        return False
-    return True
-
-
-async def mark_browser_activity() -> None:
-    """Record that an authenticated browser tab is visibly using Odysseus."""
-    global _LAST_BROWSER_ACTIVITY
-    if not _enabled():
-        return
-    cond = _condition()
-    async with cond:
-        _LAST_BROWSER_ACTIVITY = time.monotonic()
-        cond.notify_all()
-
-
-def _has_recent_browser_activity(now: float | None = None) -> bool:
-    ttl = _browser_active_seconds()
-    if ttl <= 0 or _LAST_BROWSER_ACTIVITY <= 0:
-        return False
-    return ((now if now is not None else time.monotonic()) - _LAST_BROWSER_ACTIVITY) < ttl
-
-
-def has_foreground_activity(now: float | None = None) -> bool:
-    """Return True when foreground browser/model work should stop background jobs.
-
-    Passive polling endpoints are excluded by should_track_interactive_request,
-    so active/recent request tracking is safe to use here. This matters during
-    initial page load: the heartbeat may not have landed yet, but the user is
-    already waiting on real UI requests.
-    """
-    if not _enabled():
-        return False
-    t = now if now is not None else time.monotonic()
-    if _ACTIVE_REQUESTS > 0:
-        return True
-    if _LAST_ACTIVITY > 0 and (t - _LAST_ACTIVITY) < _quiet_seconds():
-        return True
-    return _has_recent_browser_activity(t) or _has_active_chat_stream()
-
-
-def _has_active_chat_stream() -> bool:
-    """Best-effort check for foreground model work that outlives HTTP requests.
-
-    Chat/agent streams are detached from the browser SSE so a stream can keep
-    running after the request that started it has returned. Background LLM
-    tasks must still wait for those runs; otherwise helpers like email
-    auto-translate compete with the user's active chat on the same local model.
-    """
-    try:
-        from routes import chat_routes as _chat_routes
-        active_streams = getattr(_chat_routes, "_active_streams", {}) or {}
-        if active_streams:
-            return True
-    except Exception:
-        pass
-    try:
-        from src import agent_runs
-        runs = getattr(agent_runs, "_RUNS", {}) or {}
-        return any(getattr(run, "status", None) == "running" for run in runs.values())
-    except Exception:
-        return False
-
-
-@asynccontextmanager
-async def track_interactive_request(path: str = "", method: str = ""):
-    global _ACTIVE_REQUESTS, _LAST_ACTIVITY
-    if not _enabled():
-        yield
-        return
-
-    cond = _condition()
-    async with cond:
-        _ACTIVE_REQUESTS += 1
-        _LAST_ACTIVITY = time.monotonic()
-        cond.notify_all()
-    try:
-        yield
-    finally:
-        async with cond:
-            _ACTIVE_REQUESTS = max(0, _ACTIVE_REQUESTS - 1)
-            _LAST_ACTIVITY = time.monotonic()
-            cond.notify_all()
-
-
-async def wait_for_interactive_quiet(label: str = "") -> bool:
-    """Wait until foreground requests have stopped for the configured window.
-
-    Returns True if the caller had to wait at all. The label is intentionally
-    only for future logging/debugging so callers can keep their code simple.
-    """
-    if not _enabled():
-        return False
-
-    quiet = _quiet_seconds()
-    max_wait = _max_wait_seconds()
-    deadline = time.monotonic() + max_wait if max_wait > 0 else None
-    cond = _condition()
-    waited = False
-
-    while True:
-        async with cond:
-            now = time.monotonic()
-            quiet_remaining = quiet - (now - _LAST_ACTIVITY)
-            active_stream = _has_active_chat_stream()
-            browser_active = _has_recent_browser_activity(now)
-            if _ACTIVE_REQUESTS <= 0 and quiet_remaining <= 0 and not active_stream and not browser_active:
-                return waited
-
-            waited = True
-            timeout = 0.25 if (_ACTIVE_REQUESTS > 0 or active_stream or browser_active) else min(max(quiet_remaining, 0.05), 0.5)
-            if deadline is not None:
-                remaining = deadline - now
-                if remaining <= 0:
-                    return waited
-                timeout = min(timeout, remaining)
-            try:
-                await asyncio.wait_for(cond.wait(), timeout=timeout)
-            except asyncio.TimeoutError:
-                pass

src/llm_core.py

@@ -8,88 +8,13 @@ import hashlib
 import threading
 import re
 import os
-from contextlib import asynccontextmanager
 from fastapi import HTTPException
 from typing import Optional, Dict, List, Tuple
-from src.model_context import get_context_length, DEFAULT_CONTEXT, is_local_endpoint
+from src.model_context import get_context_length, DEFAULT_CONTEXT
 from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
-_LOCAL_MODEL_LOCK = asyncio.Lock()
-_LOCAL_MODEL_WAITING_FOREGROUND = 0
-_LOCAL_MODEL_CURRENT: Dict[str, object] = {}
-
-
-def _local_model_gate_enabled() -> bool:
-    return os.getenv("ODYSSEUS_LOCAL_MODEL_GATE", "true").lower() not in {"0", "false", "no", "off"}
-
-
-def _gate_workload(workload: Optional[str]) -> str:
-    return "background" if str(workload or "").lower() == "background" else "foreground"
-
-
-@asynccontextmanager
-async def _local_model_slot(target_url: str, model: str, workload: Optional[str] = None):
-    """Serialize local model traffic, with foreground chat taking priority.
-
-    Most local servers expose one GPU/CPU generation pipe even when their HTTP
-    API accepts multiple requests. Letting scheduled email/tasks and foreground
-    chat hit that pipe together creates the user-visible "streams crossed" and
-    "prompt waited behind a task" failure mode. Cloud providers are left alone.
-    """
-    if not _local_model_gate_enabled() or not is_local_endpoint(target_url):
-        yield
-        return
-
-    global _LOCAL_MODEL_WAITING_FOREGROUND
-    kind = _gate_workload(workload)
-    current_task = asyncio.current_task()
-    if kind == "foreground":
-        _LOCAL_MODEL_WAITING_FOREGROUND += 1
-        current = dict(_LOCAL_MODEL_CURRENT)
-        if current.get("workload") == "background":
-            task = current.get("task")
-            if isinstance(task, asyncio.Task) and not task.done():
-                logger.info(
-                    "[model-gate] cancelling background local model call for foreground request model=%s",
-                    model,
-                )
-                task.cancel()
-    else:
-        # Background work should not jump in while the browser/chat is active
-        # or while a foreground request is waiting to acquire the local model.
-        try:
-            from src.interactive_gate import has_foreground_activity
-        except Exception:
-            has_foreground_activity = lambda: False  # type: ignore
-        while _LOCAL_MODEL_WAITING_FOREGROUND > 0 or has_foreground_activity():
-            await asyncio.sleep(0.25)
-
-    acquired = False
-    try:
-        await _LOCAL_MODEL_LOCK.acquire()
-        acquired = True
-        if kind == "foreground":
-            _LOCAL_MODEL_WAITING_FOREGROUND = max(0, _LOCAL_MODEL_WAITING_FOREGROUND - 1)
-        _LOCAL_MODEL_CURRENT.clear()
-        _LOCAL_MODEL_CURRENT.update({
-            "task": current_task,
-            "workload": kind,
-            "url": target_url,
-            "model": model,
-            "started": time.time(),
-        })
-        yield
-    finally:
-        if kind == "foreground":
-            _LOCAL_MODEL_WAITING_FOREGROUND = max(0, _LOCAL_MODEL_WAITING_FOREGROUND - 1)
-        if acquired and _LOCAL_MODEL_LOCK.locked():
-            owner = _LOCAL_MODEL_CURRENT.get("task")
-            if owner is current_task:
-                _LOCAL_MODEL_CURRENT.clear()
-            _LOCAL_MODEL_LOCK.release()
-
 class LLMConfig:
     """Configuration constants for LLM operations."""
     DEFAULT_TIMEOUT = 30
@@ -185,18 +110,6 @@ _HARMONY_MARKERS = (
 )
 _HARMONY_MAX_MARKER_LEN = max(len(marker) for marker in _HARMONY_MARKERS)
 
-_VISIBLE_CHAT_TEMPLATE_ARTIFACT_RE = re.compile(
-    r"(?:\|end\|)+\|?assistan(?:t)?\|?"
-    r"|\|assistan(?:t)?\|"
-    r"|<\|im_start\|>\s*assistant"
-    r"|<\|im_end\|>",
-    re.IGNORECASE,
-)
-
-
-def _strip_visible_chat_template_artifacts(text: str) -> str:
-    return _VISIBLE_CHAT_TEMPLATE_ARTIFACT_RE.sub("", text or "")
-
 
 def _harmony_suffix_hold_len(text: str) -> int:
     """Return how many trailing chars could be the start of a harmony marker."""
@@ -274,75 +187,6 @@ def _stream_delta_event(text: str, *, thinking: bool = False) -> str:
         payload["thinking"] = True
     return f"data: {json.dumps(payload)}\n\n"
 
-
-_DEGENERATE_WORD_RE = re.compile(r"[A-Za-z0-9_\u0370-\u03ff\u0400-\u04ff]+")
-
-
-class _DegenerateStreamGuard:
-    """Detect local-model token collapse before it floods the UI.
-
-    Some self-hosted models fail by repeating one token forever ("Var Var Var",
-    "Summer Summer ..."). This is not a useful response and can burn context,
-    browser memory, and GPU time. Keep the guard conservative: only fire on long
-    same-token runs or a very dominant repeated token in the recent window.
-    """
-
-    def __init__(self, model: str):
-        self.model = model or "model"
-        self.last_token = ""
-        self.same_run = 0
-        self.recent_tokens: List[str] = []
-        self.total_chars = 0
-
-    def check(self, text: str) -> Optional[str]:
-        if not text:
-            return None
-        self.total_chars += len(text)
-        tokens = [t.lower() for t in _DEGENERATE_WORD_RE.findall(text) if len(t) >= 2]
-        if not tokens:
-            return None
-        for token in tokens:
-            if token == self.last_token:
-                self.same_run += 1
-            else:
-                self.last_token = token
-                self.same_run = 1
-            self.recent_tokens.append(token)
-        if len(self.recent_tokens) > 96:
-            self.recent_tokens = self.recent_tokens[-96:]
-
-        reason = None
-        if self.same_run >= 28 and self.total_chars >= 100:
-            reason = f"repeated '{self.last_token}' {self.same_run} times"
-        elif len(self.recent_tokens) >= 72:
-            top = max(set(self.recent_tokens), key=self.recent_tokens.count)
-            count = self.recent_tokens.count(top)
-            if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78:
-                reason = f"repeated '{top}' {count}/{len(self.recent_tokens)} recent tokens"
-        if not reason and len(self.recent_tokens) >= 80:
-            # Phrase loops are common on some local quantized MLX/MoE models:
-            # "Also be a software developer mode?" repeated forever will not
-            # trip the single-token guard above, but it is still a wedged
-            # generation. Require many repeats of the same 4-gram so normal
-            # prose/list formatting is not interrupted.
-            grams = [tuple(self.recent_tokens[i:i + 4]) for i in range(0, len(self.recent_tokens) - 3)]
-            if grams:
-                top_gram = max(set(grams), key=grams.count)
-                gram_count = grams.count(top_gram)
-                if gram_count >= 10:
-                    reason = f"repeated phrase '{' '.join(top_gram)}' {gram_count} times"
-
-        if not reason:
-            return None
-
-        logger.warning("[degenerate-stream] aborting model=%s reason=%s", self.model, reason)
-        message = (
-            f"Stopped generation: {self.model} started repeating tokens "
-            f"({reason}). Try a different model or lower temperature."
-        )
-        return f'event: error\ndata: {json.dumps({"status": 502, "text": message, "error": message})}\n\n'
-
-
 def _model_activity_key(url: str, model: str) -> str:
     return f"{(url or '').strip()}|{(model or '').strip()}"
 
@@ -501,18 +345,6 @@ def _normalize_ollama_url(url: str) -> str:
     return base.rstrip("/") + "/chat"
 
 
-def _normalize_openai_chat_url(url: str) -> str:
-    """Ensure an OpenAI-compatible base URL points at /chat/completions."""
-    base = (url or "").strip().rstrip("/")
-    if not base:
-        return base
-    if base.endswith("/chat/completions") or base.endswith("/completions"):
-        return base
-    if base.endswith("/models"):
-        base = base[: -len("/models")].rstrip("/")
-    return base + "/chat/completions"
-
-
 def _ollama_normalize_messages(messages: List[Dict]) -> List[Dict]:
     """Adapt Odysseus' canonical OpenAI-style messages to native Ollama /api/chat.
 
@@ -899,52 +731,6 @@ def _apply_local_cache_affinity(payload: Dict, url: str, session_id: Optional[st
     payload.setdefault("cache_prompt", True)
 
 
-def _is_local_minimax_mlx_request(url: str, model: str) -> bool:
-    """Local MLX MiniMax-family endpoints need conservative sampling defaults.
-
-    The OpenAI-compatible MLX server accepts repetition/frequency penalties.
-    Some large quantized MiniMax/MoE ports otherwise fall into visible reasoning
-    loops ("Also be...", "No.", etc.) even for trivial prompts.
-    """
-    if not model:
-        return False
-    m = model.lower()
-    if "minimax" not in m and "mini-max" not in m:
-        return False
-    try:
-        from src.model_context import is_local_endpoint
-        return is_local_endpoint(url)
-    except Exception:
-        return False
-
-
-def _apply_local_generation_stability(payload: Dict, url: str, model: str) -> None:
-    if not _is_local_minimax_mlx_request(url, model):
-        return
-    if "temperature" in payload:
-        try:
-            # MiniMax MLX quantized ports are very sensitive to chat/agent
-            # harness size. Character presets can ask for a warmer voice, but
-            # local MiniMax needs a final compatibility clamp or trivial
-            # prompts can fall into visible reasoning/repetition loops.
-            payload["temperature"] = min(float(payload.get("temperature") or 0.2), 0.2)
-        except (TypeError, ValueError):
-            payload["temperature"] = 0.2
-    payload.setdefault("top_p", 0.9)
-    payload.setdefault("top_k", 20)
-    payload.setdefault("repetition_penalty", 1.12)
-    payload.setdefault("repetition_context_size", 256)
-    payload.setdefault("frequency_penalty", 0.08)
-    payload.setdefault("frequency_context_size", 256)
-    payload.setdefault("presence_penalty", 0.02)
-    payload.setdefault("presence_context_size", 256)
-    payload.setdefault("stop", ["<|im_end|>", "<|endoftext|>", "</s>"])
-    # A max_tokens of 0 means "server default/unbounded" for many local
-    # endpoints. Keep simple chats from running forever when the model loops.
-    if not payload.get("max_tokens") and not payload.get("max_completion_tokens"):
-        payload["max_tokens"] = 2048
-
-
 def _provider_headers(provider: str, headers: Optional[Dict] = None) -> Dict[str, str]:
     h = {"Content-Type": "application/json"}
     if isinstance(headers, dict):
@@ -1575,7 +1361,6 @@ def _sanitize_llm_messages(messages: List[Dict]) -> List[Dict]:
 
     return merged
 
-
 def _normalize_anthropic_url(url: str) -> str:
     """Ensure Anthropic URL points to /v1/messages."""
     url = url.rstrip("/")
@@ -1778,7 +1563,7 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
             stream=False, num_ctx=get_context_length(url, model),
         )
     else:
-        target_url = _normalize_openai_chat_url(url)
+        target_url = url
         if provider == "copilot":
             from src.copilot import apply_request_headers
             apply_request_headers(h, messages_copy)
@@ -1792,7 +1577,6 @@ def llm_call(url: str, model: str, messages: List[Dict], temperature: float = LL
         if max_tokens and max_tokens > 0:
             tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
             payload[tok_key] = max_tokens
-        _apply_local_generation_stability(payload, target_url, model)
         if provider == "mistral" and _supports_thinking(model):
             payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
     try:
@@ -1902,7 +1686,6 @@ async def llm_call_async(
     max_retries: int = LLMConfig.MAX_RETRIES,
     prompt_type: Optional[str] = None,
     session_id: Optional[str] = None,
-    workload: str = "foreground",
 ) -> str:
     """Asynchronous LLM call using httpx with connection pooling, timeout, retry logic, and performance logging."""
     provider = _detect_provider(url)
@@ -1940,7 +1723,6 @@ async def llm_call_async(
             max_tokens=max_tokens,
             headers=headers,
             timeout=timeout,
-            workload=workload,
         ):
             event_is_error = False
             for line in str(chunk).splitlines():
@@ -1985,7 +1767,7 @@ async def llm_call_async(
             stream=False, num_ctx=get_context_length(url, model),
         )
     else:
-        target_url = _normalize_openai_chat_url(url)
+        target_url = url
         h = _provider_headers(provider, headers)
         if provider == "copilot":
             from src.copilot import apply_request_headers
@@ -2006,7 +1788,6 @@ async def llm_call_async(
         if provider == "mistral" and _supports_thinking(model):
             payload["reasoning_effort"] = _MISTRAL_REASONING_EFFORT
         _apply_local_cache_affinity(payload, url, session_id)
-        _apply_local_generation_stability(payload, target_url, model)
 
     if _is_host_dead(target_url):
         raise HTTPException(503, f"Upstream {_host_key(target_url)} marked unreachable (cooldown active)")
@@ -2017,10 +1798,9 @@ async def llm_call_async(
         attempt += 1
         start = time.time()
         try:
-            async with _local_model_slot(target_url, model, workload):
+            note_model_activity(target_url, model)
-                note_model_activity(target_url, model)
+            client = _get_http_client()
-                client = _get_http_client()
+            r = await httpx_post_kimi_aware_async(client, target_url, h, json=payload, timeout=call_timeout)
-                r = await httpx_post_kimi_aware_async(client, target_url, h, json=payload, timeout=call_timeout)
             duration = time.time() - start
             if not r.is_success:
                 friendly = _format_upstream_error(r.status_code, r.text, target_url)
@@ -2062,45 +1842,10 @@ async def llm_call_async(
                 raise HTTPException(502, f"POST {target_url} failed after {max_retries} attempts: {e}")
             await asyncio.sleep(LLMConfig.RETRY_DELAY)
 
-def _stream_target_url(url: str) -> str:
-    provider = _detect_provider(url)
-    if provider == "anthropic":
-        return _normalize_anthropic_url(url)
-    if provider == "ollama":
-        return _normalize_ollama_url(url)
-    if provider == "chatgpt-subscription":
-        return _normalize_chatgpt_subscription_url(url)
-    return _normalize_openai_chat_url(url)
-
-
 async def stream_llm(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
                      max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
                      timeout: int = LLMConfig.STREAM_TIMEOUT, prompt_type: Optional[str] = None,
-                     tools: Optional[List[Dict]] = None, session_id: Optional[str] = None,
+                     tools: Optional[List[Dict]] = None, session_id: Optional[str] = None):
-                     tool_choice_none: bool = False, workload: str = "foreground"):
-    target_url = _stream_target_url(url)
-    async with _local_model_slot(target_url, model, workload):
-        async for chunk in _stream_llm_inner(
-            url,
-            model,
-            messages,
-            temperature=temperature,
-            max_tokens=max_tokens,
-            headers=headers,
-            timeout=timeout,
-            prompt_type=prompt_type,
-            tools=tools,
-            session_id=session_id,
-            tool_choice_none=tool_choice_none,
-        ):
-            yield chunk
-
-
-async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperature: float = LLMConfig.DEFAULT_TEMPERATURE,
-                            max_tokens: int = LLMConfig.DEFAULT_MAX_TOKENS, headers: Optional[Dict] = None,
-                            timeout: int = LLMConfig.STREAM_TIMEOUT, prompt_type: Optional[str] = None,
-                            tools: Optional[List[Dict]] = None, session_id: Optional[str] = None,
-                            tool_choice_none: bool = False):
     """Stream LLM responses with improved error handling.
 
     Yields SSE chunks:
@@ -2144,7 +1889,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
         h = _provider_headers(provider, headers)
         payload = _build_chatgpt_responses_payload(model, messages_copy, temperature, max_tokens, stream=True)
     else:
-        target_url = _normalize_openai_chat_url(url)
+        target_url = url
         payload = {
             "model": model,
             "messages": messages_copy,
@@ -2160,8 +1905,6 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
             payload[tok_key] = max_tokens
         if tools:
             payload["tools"] = tools
-        elif tool_choice_none:
-            payload["tool_choice"] = "none"
         # Mistral thinking-capable models — send reasoning_effort so Mistral
         # activates thinking mode and returns structured reasoning_content.
         # Effort level is configurable via ODYSSEUS_MISTRAL_REASONING_EFFORT
@@ -2174,7 +1917,6 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
         if _is_ollama_openai_compat_url(url) and _supports_thinking(model):
             payload["think"] = False
         _apply_local_cache_affinity(payload, url, session_id)
-        _apply_local_generation_stability(payload, target_url, model)
         h = _provider_headers(provider, headers)
         if provider == "copilot":
             from src.copilot import apply_request_headers
@@ -2191,7 +1933,6 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
         yield f'event: error\ndata: {json.dumps({"error": f"Upstream {_host_key(target_url)} unreachable (cooldown active)", "status": 503})}\n\n'
         return
     note_model_activity(target_url, model)
-    degenerate_guard = _DegenerateStreamGuard(model)
 
     # ── ChatGPT Subscription / Codex Responses streaming ──
     if provider == "chatgpt-subscription":
@@ -2226,10 +1967,6 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
                     if evt == "response.output_text.delta":
                         delta = data.get("delta") or ""
                         if delta:
-                            _degenerate = degenerate_guard.check(delta)
-                            if _degenerate:
-                                yield _degenerate
-                                return
                             yield f'data: {json.dumps({"delta": delta})}\n\n'
                     elif evt == "response.completed":
                         usage = (data.get("response") or {}).get("usage") or data.get("usage") or {}
@@ -2562,19 +2299,8 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
                                                 reasoning = (reasoning + thinking_part) if reasoning else thinking_part
                                             content = text_part
                                         if reasoning:
-                                            _degenerate = degenerate_guard.check(reasoning)
-                                            if _degenerate:
-                                                yield _degenerate
-                                                return
                                             yield _stream_delta_event(reasoning, thinking=True)
                                         if content:
-                                            content = _strip_visible_chat_template_artifacts(content)
-                                            if not content:
-                                                continue
-                                            _degenerate = degenerate_guard.check(content)
-                                            if _degenerate:
-                                                yield _degenerate
-                                                return
                                             content = re.sub(r"<mm:think(\s+[^>]*)?>", r"<think\1>", content, flags=re.IGNORECASE)
                                             content = re.sub(r"</mm:think>", "</think>", content, flags=re.IGNORECASE)
                                             stripped = content.lstrip()

src/model_context.py

@@ -316,83 +316,6 @@ def _lookup_known(model: str) -> Optional[int]:
     return best_ctx
 
 
-def _model_ctx_from_entry(m: dict) -> Optional[int]:
-    """Extract a positive context window from one /models catalog entry.
-
-    Checks the common top-level fields first, then a nested meta/model_extra
-    object. Returns None when no positive window is reported.
-    """
-    if not isinstance(m, dict):
-        return None
-    for field in (
-        "context_length",
-        "context_window",
-        "max_model_len",
-        "max_context_length",
-        "max_seq_len",
-    ):
-        val = m.get(field)
-        if val and isinstance(val, (int, float)) and val > 0:
-            return int(val)
-    meta = m.get("meta") or m.get("model_extra") or {}
-    if isinstance(meta, dict):
-        # n_ctx is the actual serving context (set via -c flag in llama.cpp)
-        for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
-            val = meta.get(field)
-            if val and isinstance(val, (int, float)) and val > 0:
-                return int(val)
-    return None
-
-
-# Per-endpoint cache of the {model_id: context_length} map parsed from a
-# proxy/api catalog. api/proxy endpoints skip the /models download on every
-# lookup because a large catalog is expensive; caching the whole map lets us
-# pay that download at most once per endpoint instead of once per model.
-_catalog_ctx_cache: Dict[str, Dict[str, int]] = {}
-
-
-def _proxy_catalog_context(endpoint_url: str, model: str) -> Optional[int]:
-    """Context window for a model read from the endpoint's /models catalog.
-
-    Fetches the catalog once per endpoint and caches the full id->context map,
-    so an api/proxy endpoint serving a model that isn't in KNOWN_CONTEXT_WINDOWS
-    (e.g. a new OpenRouter model) still reports its real window instead of the
-    bare default. Returns None when the catalog can't be read or doesn't list a
-    positive window for the model.
-    """
-    cat = _catalog_ctx_cache.get(endpoint_url)
-    if cat is None:
-        from src.endpoint_resolver import build_models_url
-        try:
-            r = httpx.get(build_models_url(endpoint_url), timeout=REQUEST_TIMEOUT)
-        except Exception as e:
-            logger.debug(f"Failed to fetch proxy catalog for context length: {e}")
-            return None
-        if not r.is_success:
-            return None
-        cat = {}
-        try:
-            for m in (r.json().get("data") or []):
-                mid = m.get("id") if isinstance(m, dict) else None
-                ctx = _model_ctx_from_entry(m) if mid else None
-                if mid and ctx:
-                    cat[mid] = ctx
-        except Exception as e:
-            logger.debug(f"Failed to parse proxy catalog for context length: {e}")
-            return None
-        _catalog_ctx_cache[endpoint_url] = cat
-
-    if model in cat:
-        return cat[model]
-    # Catalog ids may carry a provider prefix (e.g. "openai/gpt-4o") while the
-    # session stores the bare id; match on the trailing segment as a fallback.
-    base = model.split("/")[-1]
-    for mid, ctx in cat.items():
-        if mid.split("/")[-1] == base:
-            return ctx
-    return None
-
-
 def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
     """Query the model API for context length. Returns (context_length, known) where
     ``known`` is False only for the bare DEFAULT_CONTEXT fallback."""
@@ -407,14 +330,6 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
         if known:
             logger.info(f"Using known context window for {model}: {known}")
             return known, True
-        # Not in the known table: read the real window from the catalog (cached
-        # once per endpoint) instead of capping every unknown model at the
-        # default — that under-reported large windows on aggregators like
-        # OpenRouter (issue #4886).
-        api_ctx = _proxy_catalog_context(endpoint_url, model)
-        if api_ctx:
-            logger.info(f"Proxy catalog reports context window for {model}: {api_ctx}")
-            return api_ctx, True
         return DEFAULT_CONTEXT, False
 
     # Try llama.cpp /slots endpoint first — reports actual serving context
@@ -455,7 +370,27 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
             for m in models_list:
                 mid = m.get("id", "")
                 if mid == model or mid.split("/")[-1] == model.split("/")[-1]:
-                    api_ctx = _model_ctx_from_entry(m)
+                    for field in (
+                        "context_length",
+                        "context_window",
+                        "max_model_len",
+                        "max_context_length",
+                        "max_seq_len",
+                    ):
+                        val = m.get(field)
+                        if val and isinstance(val, (int, float)) and val > 0:
+                            api_ctx = int(val)
+                            break
+
+                    if not api_ctx:
+                        meta = m.get("meta") or m.get("model_extra") or {}
+                        if isinstance(meta, dict):
+                            # n_ctx is the actual serving context (set via -c flag in llama.cpp)
+                            for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
+                                val = meta.get(field)
+                                if val and isinstance(val, (int, float)) and val > 0:
+                                    api_ctx = int(val)
+                                    break
                     break
     except Exception as e:
         logger.debug(f"Failed to query context length for {model}: {e}")

src/personal_docs.py

@@ -68,8 +68,6 @@ def read_text_file(path: str) -> str:
 
 def split_chunks(text: str, size: int = config.CHUNK_SIZE, overlap: int = config.CHUNK_OVERLAP) -> List[str]:
     """Split text into overlapping chunks."""
-    if not isinstance(text, str):
-        return []
     text = text.strip()
     if not text:
         return []
@@ -89,8 +87,7 @@ def split_chunks(text: str, size: int = config.CHUNK_SIZE, overlap: int = config
 
 def tokenize(s: str) -> Set[str]:
     """Tokenize string into words, excluding stop words."""
-    text = s if isinstance(s, str) else ""
+    tokens = re.findall(r"[A-Za-z0-9_\-]+", (s or "").lower())
-    tokens = re.findall(r"[A-Za-z0-9_\-]+", text.lower())
     return set(t for t in tokens if t not in config.STOP_WORDS and len(t) > 1)
 
 def load_personal_index(

src/rag_manager.py

@@ -32,9 +32,9 @@ class RAGManager:
         logger.info("RAGManager initialized as wrapper for VectorRAG")
     
     # Delegate all methods to VectorRAG
-    def search(self, query: str, k: int = 5, owner: Optional[str] = None) -> List[Dict[str, Any]]:
+    def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
         """Search for documents - delegates to VectorRAG."""
-        return self.vector_rag.search(query, k, owner=owner)
+        return self.vector_rag.search(query, k)
     
     def index_personal_documents(
         self,

src/session_search.py

@@ -207,7 +207,6 @@ def _search_like(
     )
     if not include_archived:
         q = q.filter(DBSession.archived == False)
-    q = q.filter(~DBSession.name.like("SFT trace batch%"))
     if restrict_owner:
         q = _owner_filter(q, owner, include_legacy_owner)
     rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
@@ -271,7 +270,6 @@ def _search_fts(
         WHERE chat_messages_fts MATCH :fts_query
           {archived_clause}
           {owner_clause}
-          AND s.name NOT LIKE 'SFT trace batch%'
           AND m.role IN ('user', 'assistant')
         ORDER BY bm25(chat_messages_fts), m.timestamp DESC
         LIMIT :limit

src/settings.py

@@ -136,10 +136,6 @@ DEFAULT_SETTINGS = {
     "task_model": "",
     "default_endpoint_id": "",
     "default_model": "",
-    # Optional prose style used only for normal document writing/editing.
-    # Email replies use email_writing_style instead because greetings,
-    # signatures, and mailbox identity rules are medium-specific.
-    "document_writing_style": "",
     # Ordered fallback chain for the default chat model. Each entry is
     # {"endpoint_id": "...", "model": "..."}. If the primary model fails
     # before producing output (endpoint offline / errors), the chat

src/task_action_policy.py

@@ -1,47 +0,0 @@
-"""Shared privilege policy for scheduled task actions."""
-
-from __future__ import annotations
-
-ADMIN_ONLY_TASK_ACTIONS = frozenset({
-    "run_local",
-    "run_script",
-    "ssh_command",
-    "cookbook_serve",
-})
-
-
-def is_admin_only_task_action(task_type: str | None, action: str | None) -> bool:
-    return (task_type or "llm") == "action" and (action or "") in ADMIN_ONLY_TASK_ACTIONS
-
-
-def owner_has_admin_task_privileges(owner: str | None) -> bool:
-    try:
-        from src.auth_helpers import _auth_disabled
-        if _auth_disabled():
-            return True
-    except Exception:
-        pass
-
-    if owner:
-        try:
-            from core.middleware import INTERNAL_TOOL_USER
-            if owner == INTERNAL_TOOL_USER:
-                return True
-        except Exception:
-            pass
-
-    try:
-        from core.auth import AuthManager
-        auth = AuthManager()
-        if not auth.is_configured:
-            return True
-        if not owner:
-            return False
-        return bool(auth.is_admin(owner))
-    except Exception:
-        pass
-
-    if not owner:
-        return False
-
-    return False

src/task_endpoint.py

@@ -6,7 +6,6 @@ from src.endpoint_resolver import (
     resolve_utility_fallback_candidates,
 )
 from src.llm_core import llm_call_async_with_fallback
-from src.interactive_gate import wait_for_interactive_quiet
 
 
 def resolve_task_endpoint(fallback_url=None, fallback_model=None, fallback_headers=None, owner=None):
@@ -73,6 +72,4 @@ async def task_llm_call_async(
     )
     if not candidates:
         raise RuntimeError("No LLM endpoint available for background task")
-    await wait_for_interactive_quiet("background task LLM")
-    kwargs.setdefault("workload", "background")
     return await llm_call_async_with_fallback(candidates, messages=messages, **kwargs)

src/task_scheduler.py

@@ -10,10 +10,6 @@ from datetime import datetime, timedelta, timezone
 from typing import Any, Awaitable, Callable, Dict, Tuple
 
 from core.auth import RESERVED_USERNAMES
-from src.task_action_policy import (
-    is_admin_only_task_action,
-    owner_has_admin_task_privileges,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -243,7 +239,6 @@ HOUSEKEEPING_DEFAULTS = {
     "tidy_research":        {"name": "Research Tidy",            "trigger_type": "event", "trigger_event": "research_completed", "trigger_count": 5, "schedule": None, "scheduled_time": None, "cron_expression": None, "legacy_names": ["Tidy Research"]},
     "summarize_emails":     {"name": "Email (Summary)",          "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */2 * * *", "ship_paused": True, "legacy_names": ["Tidy Email (Summary)"]},
     "draft_email_replies":  {"name": "Email AI Auto Reply",      "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */2 * * *", "ship_paused": True, "legacy_names": ["Tidy Email (Replies)", "AI Auto Reply"]},
-    "email_auto_translate": {"name": "Email Auto Translate",     "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */2 * * *", "ship_paused": True, "legacy_names": ["Auto-translate Emails", "Auto Translate Email"]},
     "extract_email_events": {"name": "Email Calendar Events",    "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 */1 * * *", "ship_paused": True, "legacy_names": ["Email → Calendar Events"]},
     "classify_events":      {"name": "Calendar Classify Events", "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 6,18 * * *", "ship_paused": True, "legacy_names": ["Classify Calendar Events"]},
     "check_email_urgency":   {"name": "Email Tags",               "schedule": "cron",  "scheduled_time": None,    "cron_expression": "0 * * * *", "ship_paused": True, "old_cron_expressions": ["*/15 * * * *"], "legacy_names": ["Email Triage", "Urgent Email"]},
@@ -691,12 +686,6 @@ class TaskScheduler:
         db = SessionLocal()
         try:
             now = _utcnow()
-            foreground_active = False
-            try:
-                from src.interactive_gate import has_foreground_activity
-                foreground_active = has_foreground_activity()
-            except Exception:
-                foreground_active = False
             async with self._executing_lock:
                 # Snapshot under the lock so we don't race with mid-iteration adds.
                 executing_snapshot = set(self._executing)
@@ -710,13 +699,8 @@ class TaskScheduler:
                 for task in due:
                     if task.id in self._executing:
                         continue
-                    if foreground_active:
-                        task.next_run = now + timedelta(minutes=15)
-                        continue
                     self._executing.add(task.id)
                     to_dispatch.append(task.id)
-                if foreground_active and due:
-                    db.commit()
             for task_id in to_dispatch:
                 asyncio.create_task(self._execute_task(task_id))
         finally:
@@ -750,26 +734,15 @@ class TaskScheduler:
 
         try:
             if bypass_model_slot or not self._task_needs_model_slot(task_id):
-                await self._execute_task_locked(
+                await self._execute_task_locked(task_id, run_id, release_executing=release_executing)
-                    task_id,
-                    run_id,
-                    release_executing=release_executing,
-                    gate_foreground=not bypass_model_slot,
-                )
                 return
 
             async with self._run_semaphore:
-                await self._execute_task_locked(
+                await self._execute_task_locked(task_id, run_id, release_executing=release_executing)
-                    task_id,
-                    run_id,
-                    release_executing=release_executing,
-                    gate_foreground=True,
-                )
         except asyncio.CancelledError:
             # If cancellation happens while queued behind the semaphore,
             # _execute_task_locked never runs and cannot update the Activity row.
             self._mark_run_aborted(task_id, run_id)
-            self._defer_immediately_due_task(task_id, delay=timedelta(minutes=15))
             raise
         finally:
             handle = self._task_handles.get(task_id)
@@ -779,36 +752,7 @@ class TaskScheduler:
                 async with self._executing_lock:
                     self._executing.discard(task_id)
 
-    def _defer_immediately_due_task(self, task_id: str, *, delay: timedelta):
+    async def _execute_task_locked(self, task_id: str, run_id: str, *, release_executing: bool = True):
-        """A queued task can be cancelled before _execute_task_locked gets a DB
-        handle. If its next_run stays in the past, the scheduler dispatches it
-        again on the next tick and spams aborted Activity rows."""
-        try:
-            from core.database import SessionLocal, ScheduledTask
-            db = SessionLocal()
-            try:
-                task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-                if (
-                    task
-                    and task.status == "active"
-                    and task.next_run is not None
-                    and task.next_run <= _utcnow()
-                ):
-                    task.next_run = _utcnow() + delay
-                    db.commit()
-            finally:
-                db.close()
-        except Exception:
-            logger.debug("Failed to defer cancelled queued task %s", task_id, exc_info=True)
-
-    async def _execute_task_locked(
-        self,
-        task_id: str,
-        run_id: str,
-        *,
-        release_executing: bool = True,
-        gate_foreground: bool = True,
-    ):
         from core.database import SessionLocal, ScheduledTask, TaskRun
 
         db = SessionLocal()
@@ -825,36 +769,6 @@ class TaskScheduler:
                     db.commit()
                 return
 
-            if (
-                is_admin_only_task_action(task.task_type, task.action)
-                and not owner_has_admin_task_privileges(task.owner)
-            ):
-                msg = f"Action '{task.action}' requires admin privileges"
-                blocked = db.query(TaskRun).filter(TaskRun.id == run_id).first()
-                if blocked:
-                    blocked.status = "error"
-                    blocked.result = msg
-                    blocked.error = msg
-                    blocked.finished_at = _utcnow()
-                task.status = "paused"
-                task.next_run = None
-                task.last_run = _utcnow()
-                logger.warning(
-                    "Paused admin-only task %s for non-admin owner %r",
-                    task_id,
-                    task.owner,
-                )
-                db.commit()
-                return
-
-            if gate_foreground:
-                waiting = db.query(TaskRun).filter(TaskRun.id == run_id).first()
-                if waiting and waiting.status == "queued":
-                    waiting.result = "Queued — waiting for Odysseus to be idle…"
-                    db.commit()
-                from src.interactive_gate import wait_for_interactive_quiet
-                await wait_for_interactive_quiet(f"scheduled task {task.name}")
-
             # Flip the run from queued → running. Reset started_at to the
             # actual execution start so queue wait time is visible from
             # created_at vs started_at if we ever surface that.
@@ -885,27 +799,6 @@ class TaskScheduler:
             # previous llm/research run's model. The executors set it once the
             # model is resolved.
             self._last_run_model = None
-            foreground_cancel = {"hit": False}
-            foreground_monitor = None
-            if gate_foreground:
-                current_task = asyncio.current_task()
-
-                async def _cancel_if_foreground_active():
-                    # Give the just-finished quiet gate a tiny grace window,
-                    # then keep enforcing "background means background" while
-                    # a long email/LLM action is already running.
-                    await asyncio.sleep(0.1)
-                    from src.interactive_gate import has_foreground_activity
-                    while True:
-                        await asyncio.sleep(0.25)
-                        if has_foreground_activity():
-                            foreground_cancel["hit"] = True
-                            logger.info("Task '%s' interrupted because Odysseus became active", task.name)
-                            if current_task:
-                                current_task.cancel()
-                            return
-
-                foreground_monitor = asyncio.create_task(_cancel_if_foreground_active())
             try:
                 if task_type == "action":
                     result, success = await self._execute_action(task, run_id=run_id)
@@ -945,22 +838,15 @@ class TaskScheduler:
                 db.commit()
                 return
             except asyncio.CancelledError:
-                msg = (
+                logger.info("Task '%s' stopped by user", task.name)
-                    "Paused because Odysseus became active"
-                    if foreground_cancel.get("hit")
-                    else "Stopped by user"
-                )
-                logger.info("Task '%s' %s", task.name, msg)
                 run_obj = db.query(TaskRun).filter(TaskRun.id == run_id).first()
                 if run_obj:
                     run_obj.status = "aborted"
-                    run_obj.error = msg
+                    run_obj.error = "Stopped by user"
-                    run_obj.result = run_obj.result or msg
+                    run_obj.result = run_obj.result or "Stopped by user"
                     run_obj.finished_at = _utcnow()
                 task.last_run = _utcnow()
-                if foreground_cancel.get("hit"):
+                if (task.trigger_type or "schedule") == "schedule":
-                    task.next_run = _utcnow() + timedelta(minutes=15)
-                elif (task.trigger_type or "schedule") == "schedule":
                     task.next_run = compute_next_run(
                         task.schedule, task.scheduled_time,
                         task.scheduled_day, task.scheduled_date,
@@ -995,13 +881,6 @@ class TaskScheduler:
                     task.next_run = None
                 db.commit()
                 return
-            finally:
-                if foreground_monitor and not foreground_monitor.done():
-                    foreground_monitor.cancel()
-                    try:
-                        await foreground_monitor
-                    except asyncio.CancelledError:
-                        pass
 
             run.finished_at = _utcnow()
 
@@ -1171,7 +1050,6 @@ class TaskScheduler:
         "learn_sender_signatures",
         "summarize_emails",
         "draft_email_replies",
-        "email_auto_translate",
         "extract_email_events",
         "classify_events",
         "tidy_sessions",
@@ -1185,7 +1063,6 @@ class TaskScheduler:
     _MODEL_BACKED_ACTIONS = frozenset({
         "summarize_emails",
         "draft_email_replies",
-        "email_auto_translate",
         "extract_email_events",
         "classify_events",
         "learn_sender_signatures",
@@ -1242,8 +1119,6 @@ class TaskScheduler:
                 self._set_run_progress(run_id, message)
 
             kwargs = {"owner": task.owner, "task_name": task.name, "progress_cb": _progress}
-            if task.prompt:
-                kwargs["prompt"] = task.prompt
             if task.action in ("run_script", "run_local", "ssh_command") and task.prompt:
                 kwargs["script" if task.action in ("run_script", "run_local") else "command"] = task.prompt
             # cookbook_serve carries its JSON config in task.prompt — feed it
@@ -1807,15 +1682,8 @@ class TaskScheduler:
 
         target = (output or "").strip()
         explicit = ""
-        account_id = ""
         if target.startswith("email:"):
             explicit = target.split(":", 1)[1].strip()
-            if "|account=" in explicit:
-                explicit, account_id = explicit.split("|account=", 1)
-                explicit = explicit.strip()
-                account_id = account_id.strip()
-            if explicit == "self":
-                explicit = ""
         elif "@" in target:
             explicit = target
 
@@ -1823,7 +1691,7 @@ class TaskScheduler:
             from routes.email_routes import _resolve_send_config
             from routes.email_helpers import _send_smtp_message
 
-            cfg = _resolve_send_config(account_id=account_id or None, owner=task.owner or "")
+            cfg = _resolve_send_config(owner=task.owner or "")
             to_addr = explicit or cfg.get("from_address") or cfg.get("smtp_user") or ""
             if not to_addr:
                 raise RuntimeError("No email recipient resolved for task output")
@@ -1891,8 +1759,6 @@ class TaskScheduler:
         # behind the primary endpoint so a downed primary won't silently yield
         # `(no output)`.
         try:
-            from src.interactive_gate import wait_for_interactive_quiet
-            await wait_for_interactive_quiet(f"agent task {task.name}")
             from src.task_endpoint import resolve_task_candidates
             _task_fallbacks = resolve_task_candidates(
                 fallback_url=endpoint_url,
@@ -1913,7 +1779,6 @@ class TaskScheduler:
             disabled_tools=disabled_tools,
             relevant_tools=relevant_tools,
             fallbacks=_task_fallbacks,
-            workload="background",
         ):
             if event_str.startswith("data: ") and not event_str.startswith("data: [DONE]"):
                 try:
@@ -2240,28 +2105,6 @@ class TaskScheduler:
         stopped = self._mark_run_aborted(task_id) or stopped
         return stopped
 
-    async def stop_background_tasks_for_foreground(self, *, reason: str = "Odysseus became active") -> int:
-        """Cancel all in-process scheduler tasks because the user is active.
-
-        This is intentionally blunt for scheduled/background work: when the
-        user opens or uses Odysseus, foreground interaction wins immediately.
-        Manual force-runs can be restarted by the user; automatic jobs will be
-        deferred by their cancellation path instead of stealing the app.
-        """
-        async with self._executing_lock:
-            task_ids = list(self._executing)
-        stopped = 0
-        for task_id in task_ids:
-            handle = self._task_handles.get(task_id)
-            if handle and not handle.done():
-                handle.cancel()
-                stopped += 1
-            if self._mark_run_aborted(task_id):
-                stopped += 1
-        if stopped:
-            logger.info("Stopped %d background scheduler task(s): %s", stopped, reason)
-        return stopped
-
     async def ensure_defaults(self, owner: str):
         """Create default housekeeping tasks for this owner (idempotent per action)."""
         from core.database import SessionLocal, ScheduledTask

src/tool_execution.py

@@ -21,12 +21,7 @@ from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
 
 
 
-from src.tool_security import (
+from src.tool_security import is_public_blocked_tool, owner_is_admin_or_single_user
-    BUILTIN_EMAIL_TOOLS,
-    email_tool_policy_names,
-    is_public_blocked_tool,
-    owner_is_admin_or_single_user,
-)
 from src.tool_policy import ToolPolicy
 from src.constants import MAX_OUTPUT_CHARS, MAX_READ_CHARS, MAX_DIFF_LINES, DATA_DIR
 from src.tool_utils import _truncate, get_mcp_manager
@@ -71,35 +66,25 @@ _SENSITIVE_FILE_PATTERNS: tuple[str, ...] = (
     "known_hosts",
 )
 
-# Case-folded views used for matching. On a case-insensitive filesystem
-# (Windows, default macOS) ".SSH/AUTHORIZED_KEYS" and ".env" resolve to the
-# same protected files as their lowercase forms, so the deny-list has to fold
-# case before comparing — the sibling resolver already normcases paths for the
-# same reason. casefold (not os.path.normcase) because normcase is a no-op on
-# POSIX, which is exactly where the macOS read-exfil path lives.
-_SENSITIVE_BASENAMES_CF: frozenset[str] = frozenset(b.casefold() for b in _SENSITIVE_BASENAMES)
-_SENSITIVE_FILE_PATTERNS_CF: frozenset[str] = frozenset(p.casefold() for p in _SENSITIVE_FILE_PATTERNS)
-
 
 def _is_sensitive_path(resolved: str) -> bool:
     """Return True if *resolved* falls under a sensitive directory or
     matches a sensitive filename — regardless of what root it sits under.
-
-    Matching is case-insensitive: on Windows / default macOS a case-variant
-    name (``.SSH``, ``AUTHORIZED_KEYS``, ``Id_Rsa``) points at the same file as
-    the lowercase form, so a case-sensitive check would let it slip past the
-    deny-list in every file tool that relies on it.
     """
-    parts = [p.casefold() for p in resolved.split(os.sep)]
+    parts = resolved.split(os.sep)
-    filename = parts[-1] if parts else ""
+    filenames: set[str] = {parts[-1]} if parts else set()
 
     # Check if any path component is a sensitive directory.
     for part in parts:
-        if part in _SENSITIVE_BASENAMES_CF:
+        if part in _SENSITIVE_BASENAMES:
             return True
 
     # Check filename against known sensitive files.
-    return filename in _SENSITIVE_FILE_PATTERNS_CF
+    for pat in _SENSITIVE_FILE_PATTERNS:
+        if pat in filenames:
+            return True
+
+    return False
 
 
 def _tool_path_roots() -> list[str]:
@@ -405,42 +390,8 @@ _MCP_ARG_PARSERS: Dict[str, Callable[[str], Dict[str, str]]] = {
 }
 
 
-# Primary argument key(s) for the legacy line-parsed tools. When a fenced
-# block's content is a JSON object carrying one of these keys, it's structured
-# inline args (the relaxed parser's ```web_search {"query": "..."}``` shape) —
-# use the object directly instead of letting the line-based parsers wrap the
-# whole JSON string as the query/url/path/prompt. Keyed off membership only
-# (the primary key never changes), so this can't drift; an unrecognized object
-# safely falls through to the line-based parser, i.e. the previous behavior.
-#
-# IMPORTANT — this only covers the MCP path. _build_mcp_args is reached via
-# _call_mcp_tool only for _MCP_TOOL_MAP tools (so an entry outside that map is
-# dead, as manage_memory was). And of these, only generate_image has a live MCP
-# server today; web_search/web_fetch/read_file/write_file have none, so they run
-# via _direct_fallback -> TOOL_HANDLERS, whose handlers decode JSON themselves
-# (see ReadFileTool/WriteFileTool/WebSearchTool/WebFetchTool). The entries here
-# are kept as defense-in-depth for if/when those servers are added. The live
-# fix for each server-less tool lives in its handler. test_write_file_inline_
-# json_args and test_mcp_json_primary_keys_are_all_live pin both halves.
-_MCP_JSON_PRIMARY_KEYS: Dict[str, tuple] = {
-    "web_search":     ("query", "queries"),
-    "web_fetch":      ("url",),
-    "read_file":      ("path",),
-    "write_file":     ("path",),
-    "generate_image": ("prompt",),
-}
-
-
 def _build_mcp_args(tool: str, content: str) -> Dict:
     """Convert fenced-block text content to structured MCP arguments."""
-    primaries = _MCP_JSON_PRIMARY_KEYS.get(tool)
-    if primaries and content.strip().startswith("{"):
-        try:
-            decoded = json.loads(content.strip())
-        except (json.JSONDecodeError, TypeError):
-            decoded = None
-        if isinstance(decoded, dict) and any(k in decoded for k in primaries):
-            return decoded
     parser = _MCP_ARG_PARSERS.get(tool)
     return parser(content) if parser else {}
 
@@ -645,12 +596,6 @@ async def _execute_tool_block_impl(
     tool = block.tool_type
     content = block.content
 
-    # The block/disable gates below must match every policy-equivalent
-    # spelling of the tool name (bare email names alias their mcp__email__
-    # form — see email_tool_policy_names), not just the spelling the model
-    # happened to emit.
-    policy_names = email_tool_policy_names(tool)
-
     # Misformatted tool call detection: model put JSON inside ```python``` (or
     # similar) without naming the tool. Common with MiniMax-style outputs.
     # Return a helpful error so the model retries with the correct format.
@@ -678,13 +623,13 @@ async def _execute_tool_block_impl(
             pass
 
     # Reject tools that the user has disabled for this request
-    if disabled_tools and not policy_names.isdisjoint(disabled_tools):
+    if disabled_tools and tool in disabled_tools:
         desc = f"{tool}: BLOCKED"
         result = {"error": f"Tool '{tool}' is disabled by user.", "exit_code": 1}
         logger.info(f"Tool blocked by user: {tool}")
         return desc, result
 
-    if tool_policy and any(tool_policy.blocks(name) for name in policy_names):
+    if tool_policy and tool_policy.blocks(tool):
         desc = f"{tool}: BLOCKED"
         result = {
             "error": f"Execution of tool '{tool}' is forbade by the active guide-only policy.",
@@ -878,51 +823,6 @@ async def _execute_tool_block_impl(
     elif tool == "vault_unlock":
         desc = "vault_unlock"
         result = await do_vault_unlock(content, owner=owner)
-    elif tool in BUILTIN_EMAIL_TOOLS:
-        # Bare email tool name from fenced-block models (e.g. Ollama) — route to MCP email server.
-        # Non-admin owners never reach here: BUILTIN_EMAIL_TOOLS ⊆ NON_ADMIN_BLOCKED_TOOLS,
-        # so is_public_blocked_tool() above already rejected them.
-        mcp = get_mcp_manager()
-        qualified = f"mcp__email__{tool}"
-        desc = f"email: {tool}"
-        if mcp:
-            _raw = content.strip()
-            args = {}
-            _args_error = None
-            if _raw:
-                # A non-empty body is always meant to be the call's arguments,
-                # and every email tool takes a JSON object. Anything that
-                # isn't one is a correctable error — NOT a silent empty-args
-                # call, which would read the DEFAULT mailbox/folder instead of
-                # the one the model meant (#3966 class). Only an EMPTY body
-                # keeps the no-arg path (e.g. ```list_email_accounts```).
-                try:
-                    parsed = json.loads(_raw)
-                except (json.JSONDecodeError, TypeError) as _je:
-                    # Covers both `{account: "work"}` (looks like JSON, bad)
-                    # and `account: work` (not JSON at all).
-                    _args_error = (
-                        f"'{tool}' arguments are not valid JSON ({_je}). "
-                        'Send a JSON object, e.g. {"account": "work"} — '
-                        "keys and string values need double quotes."
-                    )
-                else:
-                    if isinstance(parsed, dict):
-                        args = parsed
-                    else:
-                        _args_error = (
-                            f"'{tool}' arguments must be a JSON object, "
-                            'e.g. {"uid": "..."} — got a JSON array/value instead.'
-                        )
-            if _args_error is not None:
-                result = {"error": _args_error, "exit_code": 1}
-            else:
-                if owner:
-                    args = dict(args)
-                    args[_EMAIL_MCP_OWNER_ARG] = owner
-                result = await mcp.call_tool(qualified, args)
-        else:
-            result = {"error": "MCP manager not available", "exit_code": 1}
     elif tool.startswith("mcp__"):
         # MCP tool dispatch
         mcp = get_mcp_manager()

src/tool_implementations.py

@@ -36,8 +36,6 @@ def __getattr__(name):
         from src.agent_tools import admin_tools
         return getattr(admin_tools, name)
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
-
 # Cookbook (model serving) domain extracted to src/tools/cookbook.py
 # (slice 1, #4082/#4071). Re-imported here so this module stays a working
 # facade. cookbook.py pulls `_internal_headers` / `_INTERNAL_BASE` back

src/tool_index.py

@@ -105,12 +105,12 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
     "search_chats": "Search past session transcripts across chats.",
     "ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Omit `multi`/keep it false unless the question explicitly permits choosing multiple options. Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.",
     "update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.",
-    "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply <body text>` (or structured body) to open an email reply draft document without sending. USE THIS whenever the user says to write/draft a reply or tells you what to say — opening an empty draft or sending immediately is wrong. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.",
+    "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply` to open an email reply draft document without sending. To pre-fill the reply body in one shot (USE THIS whenever the user told you what to say — opening an empty draft when they asked you to write is wrong), append the body after the mode: `open_email_reply <uid> <folder> reply <body text>`. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.",
     "list_email_accounts": "List configured email accounts and default status. Use before reading or sending mail when the user mentions Gmail, work mail, custom domain mail, another mailbox, or asks to compare/check multiple inboxes.",
     "list_emails": "List emails for a folder/account, newest first, including read messages by default. Shows subject, sender, date, UID, account, and AI summary. Check inbox, find emails needing replies. Supports account from list_email_accounts for Gmail/work/custom mailboxes. For last/latest/newest email, use max_results=1 and unread_only=false.",
     "read_email": "Read the full content of a specific email by UID or Message-ID. View email body, check details. Supports account from list_email_accounts when the UID belongs to a non-default mailbox.",
     "send_email": "Send a new email via SMTP. Provide recipient, subject, body, and optional account from list_email_accounts. For replying to a thread use reply_to_email instead.",
-    "reply_to_email": "SEND a reply email immediately by UID. Do not use for write/draft/open/start reply requests; use ui_control open_email_reply with body so the user can review. Only use when the user explicitly says to send now. For send requests, use the exact UID and account from latest read_email/list_emails output; never invent UID 1. Threads automatically with In-Reply-To/References, prefixes Re:, marks original as Answered.",
+    "reply_to_email": "SEND a reply email immediately by UID. Do not use for open/start reply draft requests; use ui_control open_email_reply for those. For follow-up 'reply ...' send requests, use the exact UID and account from latest read_email/list_emails output; never invent UID 1. Threads automatically with In-Reply-To/References, prefixes Re:, marks original as Answered.",
     "archive_email": "Move an email out of the inbox into the Archive folder. Use after handling messages you want to keep but get out of the way.",
     "delete_email": "Delete an email — moves to Trash by default, or expunges permanently with permanent=true.",
     "mark_email_read": "Mark an email as read or unread by toggling the \\Seen flag.",
@@ -118,7 +118,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
     "resolve_contact": "Look up a contact's email address by name. Searches CardDAV address book and sent email history. Use when the user says 'message [name]', 'email [name]', or 'send to [name]' without an email address.",
     "manage_contact": "Save / update / delete / list address-book contacts (CardDAV). Use for info about ANOTHER person — name, email, phone, postal address. Args: action=list|add|update|delete, name, email, phones, address, uid (from list). For 'save this for <person>' / address pastes / phone numbers next to a name, this is the right tool — NOT manage_memory. Do NOT use for facts about the USER ('my name is X'); those are manage_memory.",
     "manage_notes": "Create and manage notes and checklists (Google Keep-style). ALWAYS use this for note/todo/checklist/reminder creation — NEVER hit /api/notes via app_api. Accepts natural-language `due_date` like 'tomorrow at 9am' or '11pm today' (parsed in the USER'S timezone). The due_date IS the reminder — it fires a notification at that time, so do NOT also create a calendar event for the same reminder. Set colors, labels, pin, archive. Do NOT use manage_memory for note content.",
-    "manage_calendar": "Calendar event management: list, create, update, delete. Each event can carry a tag/category (event_type — work/personal/health/travel/meal/social/admin/other) and importance (low/normal/high/critical). Resolve today/tomorrow using the Current date and time context, then use ISO datetimes in the user's local wall time; supports all-day events. Use rrule only for explicit recurrence; for update_event pass rrule='' to remove repeats. For event reminders/alarms, pass reminder_minutes; this creates the Notes reminder, so do not also call manage_notes for the same reminder.",
+    "manage_calendar": "Calendar event management: list, create, update, delete. Each event can carry a tag/category (event_type — work/personal/health/travel/meal/social/admin/other) and importance (low/normal/high/critical). Resolve today/tomorrow using the Current date and time context, then use ISO datetimes in the user's local wall time; supports all-day events. For event reminders/alarms, pass reminder_minutes; this creates the Notes reminder, so do not also call manage_notes for the same reminder.",
     "download_model": "Download a HuggingFace model to a local or remote server. Specify repo_id (e.g. 'Qwen/Qwen3-8B'), optional server host, and optional include filter for specific files.",
     "serve_model": "Start serving a model with vLLM, SGLang, llama.cpp, Ollama, or Diffusers. cmd MUST start with the binary directly — e.g. `vllm serve /mnt/HADES/models/Qwen3.5-397B-A17B-AWQ --port 8003 --tensor-parallel-size 8 …`. NEVER prefix with `cd …`, `source …`, or chain with `&&`/`||` — those get rejected by the validator. The venv activation (env_prefix) and CUDA env are added automatically from the target host's saved settings. For image/inpainting/diffusion use python3 scripts/diffusion_server.py --model <repo> --port 8100. After launch, call list_served_models for readiness/errors and retry suggestions. If serve_model fails with 'Invalid characters in cmd', simplify to the bare binary + args.",
     "list_served_models": "List currently running model servers in the Cookbook — shows status (loading, ready, idle, error), model name, port, throughput, and serve failure diagnosis/retry suggestions. Use when the user asks 'what's running', 'show my cookbook', 'which models are up', 'what's serving'.",
@@ -405,10 +405,8 @@ class ToolIndex:
             {"chat_with_model", "ask_teacher", "list_models"},
         # Deep research intent (incl. common typo "reserach")
         frozenset({"web search", "search the web", "search online", "look up",
-                   "find info online", "find information online",
+                   "google", "latest", "current", "news", "weather",
-                   "find info", "find information", "online about",
+                   "forecast", "stock price", "price of"}):
-                   "on the internet", "google", "latest", "current", "news",
-                   "weather", "forecast", "stock price", "price of"}):
             {"web_search", "web_fetch"},
         frozenset({"research", "reserach", "reasearch", "look into", "investigate",
                    "deep dive", "deep research", "find out about", "study up on",

src/tool_parsing.py

@@ -10,10 +10,9 @@ import bisect
 import json
 import logging
 import re
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 from src.agent_tools import ToolBlock, TOOL_TAGS
-from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
@@ -21,63 +20,12 @@ logger = logging.getLogger(__name__)
 # Regex patterns
 # ---------------------------------------------------------------------------
 
-# Pattern 1: ```bash ... ``` fenced code blocks. The tag may be followed by a
+# Pattern 1: ```bash ... ``` fenced code blocks
-# newline (classic form) or by inline JSON args on the same line
-# (```list_email_accounts {}). The same-line part is captured separately
-# (group 2) and judged by _fenced_tool_call below — the regex alone only
-# requires it to start with { or [; anything else after the tag is a Markdown
-# info string (```python title="example.py") and the fence never matches.
-# (?![\w-]) keeps the alternation from prefix-matching longer fence tags:
-# without it, ```python3 would match as tool "python" with content "3\n..."
-# and execute as code.
 _TOOL_BLOCK_RE = re.compile(
-    r"```(" + "|".join(TOOL_TAGS) + r")(?![\w-])"
+    r"```(" + "|".join(TOOL_TAGS) + r")\s*\n([\s\S]*?)```",
-    r"[ \t]*([{\[][^\n]*?)?[ \t]*(?=\r?\n|```)\r?\n?([\s\S]*?)```",
     re.IGNORECASE,
 )
 
-# Tags whose fenced content is raw code, not JSON args. Same-line text after
-# these tags is Markdown fence metadata on a real language (```bash {title=
-# "setup"}), never inline tool args — only the classic tag-then-newline form
-# executes for them.
-_CODE_FENCE_TAGS = frozenset({"bash", "python"})
-
-
-def _fenced_tool_call(m) -> Optional[Tuple[str, str]]:
-    """Classify a Pattern-1 fence match: (tag, content) when it is an
-    executable tool call, None when the fence must stay display text.
-
-    Shared by parse_tool_blocks and strip_tool_blocks so the execute and
-    display decisions can never disagree: a fence that doesn't execute is
-    never stripped, and vice versa.
-
-    Same-line text after the tag only counts as inline tool args when the
-    tag's tool takes JSON args (not a code tag) AND the text is valid
-    standalone JSON. ```bash {title="setup"} and ```python {"x": 1} are
-    fence attributes on real languages, and {title="x"} on any tag is
-    metadata, not arguments — all of those stay visible and inert.
-    """
-    tag = m.group(1).lower()
-    inline = (m.group(2) or "").strip()
-    body = (m.group(3) or "").strip()
-    if not inline:
-        return tag, body
-    if tag in _CODE_FENCE_TAGS:
-        return None
-    # Inline args may continue onto following lines (a JSON object opened on
-    # the tag line); the combined text must parse as JSON or nothing runs.
-    content = f"{inline}\n{body}" if body else inline
-    try:
-        json.loads(content)
-    except (ValueError, TypeError):
-        return None
-    return tag, content
-
-
-def _strip_executed_fence(m) -> str:
-    """re.sub callback: remove only fences that parse as tool calls."""
-    return "" if _fenced_tool_call(m) is not None else m.group(0)
-
 # Pattern 2: [TOOL_CALL] ... [/TOOL_CALL] blocks (some models use this format)
 # Matches: {tool => "shell", args => {--command "ls -la"}} etc.
 _TOOL_CALL_RE = re.compile(
@@ -166,34 +114,6 @@ _TOOL_CODE_RE = re.compile(
 _TOOL_CODE_OPEN_RE = re.compile(r"<tool_code>\s*\{", re.IGNORECASE)
 _TOOL_CODE_CLOSE_RE = re.compile(r"\}\s*</tool_code>", re.IGNORECASE)
 
-# Pattern 4b: Gemma-style <|tool_call|> call:tool_name{args} <tool_call|>
-_GEMMA_TOOL_CALL_RE = re.compile(
-    r"<\|?tool_call\|?>\s*call:([\w\d_-]+)\s*(\{[\s\S]*?\})\s*<\|?tool_call\|?>",
-    re.IGNORECASE,
-)
-
-# Pattern 4c: Open-function wrapper emitted by some local MLX/Exo models.
-# Example:
-#   <function_model>
-#   <function_call>web_search</function_call>
-#   <parameters>{"query":"Sweden news today"}</parameters>
-#   </function_model>
-_FUNCTION_MODEL_OPEN_RE = re.compile(r"<function_model>\s*", re.IGNORECASE)
-_FUNCTION_MODEL_CLOSE_RE = re.compile(r"</function_model>", re.IGNORECASE)
-_FUNCTION_MODEL_NAME_RE = re.compile(
-    r"<function_call>\s*([A-Za-z_][\w-]*)\s*</function_call>",
-    re.IGNORECASE,
-)
-_FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE)
-_FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE)
-_QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE)
-_QWEN_BARE_MARKER_RE = re.compile(
-    r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|"
-    r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)",
-    re.IGNORECASE,
-)
-
-
 # Pattern 5: DeepSeek DSML markup leaking into content. When deepseek
 # models can't emit structured tool_calls (e.g. we sent no tool schemas
 # that round, or the API didn't parse them), they fall back to raw
@@ -336,17 +256,6 @@ _RAW_WEB_JSON_TOOL_RE = re.compile(
 )
 _RAW_WEB_JSON_ALLOWED_KEYS = {"query", "queries", "time_filter", "freshness", "max_pages"}
 
-# Narrow rescue for models that ignore native tool calling and print the UI
-# command as plain text. Keep this intentionally tiny: open-panel is a harmless
-# frontend event, while broad plain-text parsing of shell/doc/email tools would
-# be unsafe.
-_PLAIN_UI_OPEN_PANEL_RE = re.compile(
-    r"(?im)^\s*(?:`{1,3})?\s*ui_control\s+open_panel\s+"
-    r"(documents?|library|gallery|images?|email|inbox|mail|sessions?|chats?|history|"
-    r"notes?|brain|memor(?:y|ies)|skills?|settings|preferences|cookbook|models?)"
-    r"\s*(?:`{1,3})?\s*$"
-)
-
 
 # ---------------------------------------------------------------------------
 # Parsing functions
@@ -587,205 +496,6 @@ def _parse_raw_web_json_lookup(text: str) -> Optional[tuple[ToolBlock, tuple[int
                 return block, (start, start + end)
     return None
 
-
-def _looks_like_openai_tool_call_blob(value) -> bool:
-    """Return True for raw OpenAI-style tool-call JSON leaked as text."""
-    if isinstance(value, list):
-        return bool(value) and all(_looks_like_openai_tool_call_blob(item) for item in value)
-    if not isinstance(value, dict):
-        return False
-    fn = value.get("function")
-    if isinstance(fn, dict) and isinstance(fn.get("name"), str):
-        return True
-    return False
-
-
-def _raw_openai_tool_call_to_block(value) -> Optional[ToolBlock]:
-    if isinstance(value, list):
-        for item in value:
-            block = _raw_openai_tool_call_to_block(item)
-            if block:
-                return block
-        return None
-    if not isinstance(value, dict):
-        return None
-    fn = value.get("function")
-    if not isinstance(fn, dict):
-        return None
-    name = str(fn.get("name") or "").strip()
-    if not name:
-        return None
-    tool_type = _TOOL_NAME_MAP.get(name, name)
-    raw_args = fn.get("arguments") or {}
-    try:
-        args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
-    except (json.JSONDecodeError, TypeError):
-        args = {}
-    if not isinstance(args, dict):
-        args = {}
-    # Common local-model typo seen in raw OpenAI JSON leaks.
-    if "text" not in args and "tex" in args:
-        args["text"] = args.get("tex")
-
-    if tool_type.startswith("mcp__"):
-        return ToolBlock(tool_type, json.dumps(args) if args else "{}")
-    if name in BUILTIN_EMAIL_TOOLS:
-        return ToolBlock(f"mcp__email__{name}", json.dumps(args) if args else "{}")
-    if tool_type not in TOOL_TAGS:
-        return None
-
-    if tool_type == "bash":
-        content = args.get("command", "")
-    elif tool_type == "python":
-        content = args.get("code", "")
-    elif tool_type == "web_search":
-        content = args.get("query", "")
-        queries = args.get("queries")
-        if not content and isinstance(queries, list) and queries:
-            content = str(queries[0])
-        elif not content and queries:
-            content = str(queries)
-        tf = args.get("time_filter")
-        if content and isinstance(tf, str) and tf in ("day", "week", "month", "year"):
-            content = json.dumps({"query": content, "time_filter": tf})
-    elif tool_type == "web_fetch":
-        content = args.get("url") or args.get("domain") or ""
-    elif tool_type == "read_file":
-        content = json.dumps(args) if (args.get("offset") or args.get("limit")) else args.get("path", "")
-    elif tool_type in ("grep", "glob", "ls", "edit_file"):
-        content = json.dumps(args) if args else "{}"
-    elif tool_type == "write_file":
-        content = args.get("path", "") + "\n" + args.get("content", "")
-    elif tool_type == "create_document":
-        parts = [args.get("title", "Untitled")]
-        if args.get("language"):
-            parts.append(args["language"])
-        parts.append(args.get("content", ""))
-        content = "\n".join(parts)
-    elif tool_type == "update_document":
-        content = args.get("content", "")
-    elif tool_type in ("edit_document", "suggest_document"):
-        marker = "SUGGEST" if tool_type == "suggest_document" else "REPLACE"
-        blocks = []
-        for edit in args.get("suggestions" if tool_type == "suggest_document" else "edits", []) or []:
-            if not isinstance(edit, dict):
-                continue
-            block = f'<<<FIND>>>\n{edit.get("find", "")}\n<<<{marker}>>>\n{edit.get("replace", "")}'
-            if tool_type == "suggest_document":
-                block += f'\n<<<REASON>>>\n{edit.get("reason", "")}'
-            blocks.append(block + "\n<<<END>>>")
-        content = "\n".join(blocks)
-    elif tool_type == "search_chats":
-        content = args.get("query", "")
-    elif tool_type == "chat_with_model":
-        content = args.get("model", "") + "\n" + args.get("message", "")
-    elif tool_type == "create_session":
-        content = args.get("name", "Untitled") + "\n" + args.get("model", "")
-    elif tool_type == "list_sessions":
-        content = args.get("filter", "")
-    elif tool_type == "send_to_session":
-        content = args.get("session_id", "") + "\n" + args.get("message", "")
-    elif tool_type == "pipeline":
-        content = json.dumps({"steps": args.get("steps", [])})
-    elif tool_type == "manage_session":
-        action = args.get("action", "")
-        if action == "list":
-            keyword = args.get("keyword", "") or args.get("value", "")
-            content = "list" + (("\n" + keyword) if keyword and keyword.lower() != "current" else "")
-        else:
-            content = action + "\n" + args.get("session_id", "current")
-            if args.get("value"):
-                content += "\n" + args["value"]
-    elif tool_type == "manage_memory":
-        action = args.get("action", "")
-        if action == "add":
-            content = "add\n" + str(args.get("text", ""))
-            if args.get("category"):
-                content += "\n" + str(args["category"])
-        elif action == "edit":
-            content = "edit\n" + str(args.get("memory_id", "")) + "\n" + str(args.get("text", ""))
-        elif action == "delete":
-            content = "delete\n" + str(args.get("memory_id", ""))
-        elif action == "search":
-            content = "search\n" + str(args.get("text", ""))
-        elif action == "list":
-            content = "list" + (("\n" + str(args["category"])) if args.get("category") else "")
-        else:
-            content = action
-    elif tool_type == "ui_control":
-        action = args.get("action", "")
-        name_arg = args.get("name", "")
-        value = args.get("value", "")
-        if action == "open_panel":
-            content = f"open_panel {name_arg or value}"
-        elif action == "toggle":
-            content = f"toggle {name_arg} {value}"
-        else:
-            content = action
-    elif tool_type in ("manage_tasks", "manage_skills", "api_call", "manage_endpoints",
-                       "manage_mcp", "manage_webhooks", "manage_tokens",
-                       "manage_documents", "manage_settings", "manage_notes",
-                       "manage_research", "manage_bg_jobs"):
-        content = json.dumps(args)
-    elif tool_type in ("get_workspace", "list_models"):
-        content = args.get("filter", "") if tool_type == "list_models" else ""
-    else:
-        content = json.dumps(args) if args else ""
-    return ToolBlock(tool_type, str(content or ""))
-
-
-def _parse_raw_openai_tool_call_json(text: str) -> Optional[ToolBlock]:
-    if not isinstance(text, str) or '"function"' not in text:
-        return None
-    decoder = json.JSONDecoder()
-    for match in re.finditer(r"[\[{]", text):
-        try:
-            parsed, _end = decoder.raw_decode(text[match.start():])
-        except json.JSONDecodeError:
-            continue
-        block = _raw_openai_tool_call_to_block(parsed)
-        if block:
-            return block
-    return None
-
-
-def _strip_raw_openai_tool_call_json(text: str) -> str:
-    """Strip raw JSON tool calls such as {"function": {...}, "type": "function"}.
-
-    Some local models emit native tool-call JSON into assistant text. The agent
-    can still parse/execute it through the native path, but the raw payload must
-    not render or persist as prose.
-    """
-    if not isinstance(text, str) or '"function"' not in text:
-        return text
-    decoder = json.JSONDecoder()
-    pieces = []
-    pos = 0
-    changed = False
-    for match in re.finditer(r"[\[{]", text):
-        start = match.start()
-        if start < pos:
-            continue
-        try:
-            parsed, rel_end = decoder.raw_decode(text[start:])
-        except json.JSONDecodeError:
-            continue
-        end = start + rel_end
-        if not _looks_like_openai_tool_call_blob(parsed):
-            continue
-        pieces.append(text[pos:start])
-        pos = end
-        changed = True
-        # Common broken local-model suffix: a standalone ] before a role marker.
-        while pos < len(text) and text[pos] in " \t\r\n":
-            pos += 1
-        if pos < len(text) and text[pos] == "]":
-            pos += 1
-    if not changed:
-        return text
-    pieces.append(text[pos:])
-    return "".join(pieces)
-
 def _parse_tool_call_block(raw: str) -> Optional[ToolBlock]:
     """Parse a [TOOL_CALL] block into a ToolBlock.
 
@@ -1070,58 +780,6 @@ def _parse_tool_code_block(raw: str) -> Optional[ToolBlock]:
         return ToolBlock(tool_name, content.strip())
     return None
 
-def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
-    """Parse a Gemma-style call:tool_name{...} block into a ToolBlock."""
-    tool_name = tool_name.strip().lower().replace("-", "_")
-    body = body.strip()
-    if not body:
-        return None
-
-    # Replace custom Gemma string delimiters with standard quotes
-    body = body.replace('<|"|>', '"').replace('<|"', '"').replace('"|>', '"')
-
-    # Try standard JSON parsing
-    params = {}
-    try:
-        params = json.loads(body)
-        if not isinstance(params, dict):
-            params = {}
-    except json.JSONDecodeError:
-        # Try unquoted keys repair: e.g. {query: "..."} -> {"query": "..."}
-        try:
-            repaired = re.sub(r'([{,]\s*)(\w+)\s*:', r'\1"\2":', body)
-            params = json.loads(repaired)
-            if not isinstance(params, dict):
-                params = {}
-        except Exception:
-            # Simple regex key-value extraction fallback
-            params = {}
-            for m in re.finditer(r'(\w+)\s*:\s*["\']?(.*?)["\']?(?=\s*,\s*\w+\s*:|\s*\})', body):
-                k = m.group(1)
-                v = m.group(2).strip()
-                params[k] = v
-
-    from src.tool_schemas import function_call_to_tool_block
-    return function_call_to_tool_block(tool_name, json.dumps(params))
-
-
-def _parse_function_model_call(body: str) -> Optional[ToolBlock]:
-    """Parse <function_model><function_call>tool</...><parameters>...</...>."""
-    name_match = _FUNCTION_MODEL_NAME_RE.search(body or "")
-    if not name_match:
-        return None
-    tool_name = name_match.group(1).strip().lower().replace("-", "_")
-    params = "{}"
-    for _ms, inner_start, inner_end, _me in _iter_delimited(
-        body,
-        _FUNCTION_MODEL_PARAMS_OPEN_RE,
-        _FUNCTION_MODEL_PARAMS_CLOSE_RE,
-    ):
-        params = body[inner_start:inner_end].strip() or "{}"
-        break
-    from src.tool_schemas import function_call_to_tool_block
-    return function_call_to_tool_block(tool_name, params)
-
 
 def _iter_delimited(text, open_re, close_re):
     """Yield ``(match_start, inner_start, inner_end, match_end)`` for each
@@ -1265,20 +923,9 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]:
     # Pattern 1: fenced code blocks (skipped when `skip_fenced` — see docstring).
     if not skip_fenced:
         for m in _TOOL_BLOCK_RE.finditer(text):
-            call = _fenced_tool_call(m)
+            tag = m.group(1).lower()
-            if call is None:
+            content = m.group(2).strip()
-                continue
-            tag, content = call
             if not content:
-                # An empty fence is still an unambiguous call for the email
-                # tools — ```list_email_accounts``` with no body is a shape
-                # local models really emit for no-arg tools. Dispatch with
-                # empty args and let the tool's own validation answer;
-                # silently dropping the call left models concluding email was
-                # broken. Other tags (bash, python, ...) keep skipping: empty
-                # content is nothing to run.
-                if tag in BUILTIN_EMAIL_TOOLS:
-                    blocks.append(ToolBlock(tag, ""))
                 continue
             # If a code block's content is an <invoke> XML call (some models wrap
             # tool calls in ```python or ```xml fences), parse the invoke instead.
@@ -1365,45 +1012,12 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]:
             if block:
                 blocks.append(block)
 
-    # Pattern 4b: Gemma-style <|tool_call|> blocks
-    if not blocks:
-        for m in _GEMMA_TOOL_CALL_RE.finditer(text):
-            tool_name = m.group(1)
-            body = m.group(2)
-            block = _parse_gemma_tool_call(tool_name, body)
-            if block:
-                blocks.append(block)
-
-    # Pattern 4c: <function_model> wrapper from local MLX/Exo models.
-    if not blocks:
-        for _ms, inner_start, inner_end, _me in _iter_delimited(
-            text, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE
-        ):
-            block = _parse_function_model_call(text[inner_start:inner_end])
-            if block:
-                blocks.append(block)
-
-    # Pattern 4d: raw OpenAI-style tool-call JSON leaked as assistant text.
-    # Example: {"function":{"arguments":"{\"action\":\"add\"}","name":"manage_memory"},"type":"function"}
-    if not blocks:
-        block = _parse_raw_openai_tool_call_json(text)
-        if block:
-            blocks.append(block)
-
     # Pattern 6: local text-model web_search call leaked as prose + bare JSON.
     if not blocks and not skip_fenced:
         raw_web_json = _parse_raw_web_json_lookup(text)
         if raw_web_json:
             blocks.append(raw_web_json[0])
 
-    # Pattern 7: plain `ui_control open_panel notes` line. This commonly comes
-    # from weaker native-tool models after reading the tool docs but failing to
-    # emit the actual structured call.
-    if not blocks:
-        m = _PLAIN_UI_OPEN_PANEL_RE.search(text)
-        if m:
-            blocks.append(ToolBlock("ui_control", f"open_panel {m.group(1).lower()}"))
-
     return blocks
 
 
@@ -1423,10 +1037,7 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str:
     # Normalize DSML first so its markup gets stripped by the <invoke>
     # / <tool_call> removers below instead of leaking to the user.
     text = _normalize_dsml(text)
-    # Keep the executed-vs-illustrative fence distinction (only strip fences
+    cleaned = text if skip_fenced else _TOOL_BLOCK_RE.sub('', text)
-    # that actually dispatched; leave example fences from native models inert
-    # but visible), then remove [TOOL_CALL]{...}[/TOOL_CALL] markup.
-    cleaned = text if skip_fenced else _TOOL_BLOCK_RE.sub(_strip_executed_fence, text)
     # Forward-only removal mirrors parse_tool_blocks: _strip_delimited pairs each
     # opener with a later closer and stops when none is reachable, so untrusted
     # output can't drive the O(n^2) lazy-rescan (ReDoS); see _iter_delimited.
@@ -1435,17 +1046,11 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str:
     cleaned = _strip_delimited(cleaned, _XML_TOOL_CALL_OPEN_RE, _XML_TOOL_CALL_CLOSE_RE)
     cleaned = _XML_OPEN_TOOL_CALL_RE.sub('', cleaned)
     cleaned = _strip_delimited(cleaned, _TOOL_CODE_OPEN_RE, _TOOL_CODE_CLOSE_RE)
-    cleaned = _GEMMA_TOOL_CALL_RE.sub('', cleaned)
-    cleaned = _strip_delimited(cleaned, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE)
-    cleaned = _strip_raw_openai_tool_call_json(cleaned)
-    cleaned = _QWEN_ROLE_MARKER_RE.sub('', cleaned)
-    cleaned = _QWEN_BARE_MARKER_RE.sub(' ', cleaned)
     if not skip_fenced:
         raw_web_json = _parse_raw_web_json_lookup(cleaned)
         if raw_web_json:
             _, (start, end) = raw_web_json
             cleaned = cleaned[:start] + cleaned[end:]
-    cleaned = _PLAIN_UI_OPEN_PANEL_RE.sub("", cleaned)
     # Strip bare <invoke> blocks not wrapped in <tool_call>
     cleaned = _strip_bare_invoke_markup(cleaned)
     cleaned = re.sub(r'\n{3,}', '\n\n', cleaned)

src/tool_policy.py

@@ -16,39 +16,6 @@ GUIDE_ONLY_DIRECTIVE = (
     "output they will produce locally."
 )
 
-WEB_TOOL_NAMES = frozenset({"web_search", "web_fetch"})
-
-
-def tool_toggle_enabled(value: object) -> bool:
-    """Return true only for explicit true-like tool toggle values."""
-
-    return str(value).lower() == "true"
-
-
-def tool_toggle_explicitly_denied(value: object) -> bool:
-    """Return true when a caller explicitly supplied a non-true toggle value."""
-
-    return value is not None and not tool_toggle_enabled(value)
-
-
-def is_web_search_explicitly_denied(allow_web_search: object) -> bool:
-    """Whether the web-search agent toggle was explicitly set to false."""
-
-    return tool_toggle_explicitly_denied(allow_web_search)
-
-
-def web_search_enabled_for_turn(allow_web_search: object, use_web: object = None) -> bool:
-    """Return true only when this request explicitly enables web search.
-
-    Agent mode sends ``allow_web_search``; chat-mode pre-search sends
-    ``use_web``. If both are present, an explicit ``allow_web_search=false``
-    wins so a stale or conflicting intent path cannot re-enable web tools.
-    """
-
-    if is_web_search_explicitly_denied(allow_web_search):
-        return False
-    return tool_toggle_enabled(allow_web_search) or tool_toggle_enabled(use_web)
-
 
 _COMMON_TOOL_NAMES = {
     "api_call",

src/tool_schemas.py

@@ -14,19 +14,9 @@ from typing import Optional
 
 from src.agent_tools import ToolBlock, TOOL_TAGS
 from src.tool_parsing import _TOOL_NAME_MAP
-from src.tool_security import BUILTIN_EMAIL_TOOLS
 
 logger = logging.getLogger(__name__)
 
-
-_REQUIRED_NATIVE_TOOL_ARGS = {
-    "web_search": ("query", "queries"),
-    "web_fetch": ("url",),
-    "read_file": ("path",),
-    "write_file": ("path",),
-    "edit_file": ("path",),
-}
-
 # ---------------------------------------------------------------------------
 # OpenAI-compatible function tool schemas
 # ---------------------------------------------------------------------------
@@ -196,7 +186,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "create_document",
-            "description": "Create a new document in the editor panel. Use this when the user asks to write, create, build, make, or generate code, scripts, programs, games, apps, or any long-form or structured content that is more than a short paragraph, AND there is no already-open document/email draft that the request refers to. If an email compose draft is open, edit that draft instead of creating another document. NEVER put large generated content directly in chat — use this tool instead.",
+            "description": "Create a new document in the editor panel. Use this when the user asks to write, create, build, or generate code, scripts, programs, games, apps, or any substantial content (>15 lines) AND there is no already-open document/email draft that the request refers to. If an email compose draft is open, edit that draft instead of creating another document. NEVER put large code blocks directly in chat — use this tool instead.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -335,7 +325,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "send_to_session",
-            "description": "Send a new message to an existing live chat and get that chat model's response. Do not use this to retrieve, read, summarize, or inspect old chats; use search_chats or list_sessions for past chat evidence.",
+            "description": "Send a message to an existing chat and get the model's response. The chat keeps its conversation history.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -425,7 +415,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "ui_control",
-            "description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), open_email_reply (open an email reply draft document; DOES NOT send. For 'write/draft a reply saying X', include body with the drafted reply), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation). When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.",
+            "description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), open_email_reply (open an email reply draft document; does NOT send), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation). When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -436,7 +426,6 @@ FUNCTION_TOOL_SCHEMAS = [
                     "uid": {"type": "string", "description": "Email UID for open_email_reply"},
                     "folder": {"type": "string", "description": "Email folder for open_email_reply (default INBOX)"},
                     "mode": {"type": "string", "description": "Reply draft mode for open_email_reply: reply, reply-all, or ai-reply"},
-                    "body": {"type": "string", "description": "For open_email_reply: reply body to pre-fill. Required whenever the user told you what the reply should say. Opens a draft, does not send."},
                     "colors": {"type": "object", "description": "For create_theme: the theme colors",
                                "properties": {
                                    "bg": {"type": "string", "description": "Background color (hex, e.g. #1a1a2e)"},
@@ -549,7 +538,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "manage_calendar",
-            "description": "Manage calendar events: list events in a date range, create, update, delete. Each event can carry a tag/category (event_type) and importance level. Resolve relative dates like today/tomorrow against the 'Current date and time' system context, then pass ISO 8601 datetimes in the user's local wall time; for all-day events set all_day=true and pass YYYY-MM-DD. For event reminders/alarms, pass reminder_minutes; the tool creates the Odysseus note reminder, so do not also call manage_notes for the same reminder. Do not set rrule for single-occurrence requests such as 'next Wednesday only'; use rrule only when the user explicitly wants recurrence.",
+            "description": "Manage calendar events: list events in a date range, create, update, delete. Each event can carry a tag/category (event_type) and importance level. Resolve relative dates like today/tomorrow against the 'Current date and time' system context, then pass ISO 8601 datetimes in the user's local wall time; for all-day events set all_day=true and pass YYYY-MM-DD. For event reminders/alarms, pass reminder_minutes; the tool creates the Odysseus note reminder, so do not also call manage_notes for the same reminder.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -565,12 +554,12 @@ FUNCTION_TOOL_SCHEMAS = [
                     "uid": {"type": "string", "description": "Event UID (for update/delete)"},
                     "calendar_href": {"type": "string", "description": "Specific calendar URL (optional; defaults to first calendar)"},
                     "calendar": {"type": "string", "description": "Filter list_events by calendar name or href"},
-                    "start": {"type": "string", "description": "list_events range start (ISO datetime). Use this for month/week requests after resolving the date range; do not pass a loose query string. Prefer start; backend also accepts start_time, start_date, range_start, from, dtstart, since."},
+                    "start": {"type": "string", "description": "list_events range start (ISO datetime); defaults to today. Prefer start; backend also accepts start_date, range_start, from, dtstart, since."},
-                    "end": {"type": "string", "description": "list_events range end (ISO datetime). Use this for month/week requests after resolving the date range; defaults to +14 days only when no range is requested. Prefer end; backend also accepts end_time, end_date, range_end, to, dtend, until."},
+                    "end": {"type": "string", "description": "list_events range end (ISO datetime); defaults to +14 days. Prefer end; backend also accepts end_date, range_end, to, dtend, until."},
                     "event_type": {"type": "string", "description": "Tag / category for the event. Common values: work, personal, health, travel, meal, social, admin, other. Aliases accepted: tag, category, type."},
                     "importance": {"type": "string", "enum": ["low", "normal", "high", "critical"], "description": "Priority level (defaults to 'normal')"},
                     "reminder_minutes": {"type": "integer", "description": "For create_event: create an Odysseus reminder this many minutes before the event, e.g. 5 for 'reminder 5 min before'."},
-                    "rrule": {"type": "string", "description": "Recurrence rule in iCalendar RRULE format, e.g. 'FREQ=WEEKLY;BYDAY=MO' for weekly on Monday. Use with create_event or update_event. For update_event, pass an explicit empty string to remove recurrence and make the event single-occurrence."}
+                    "rrule": {"type": "string", "description": "Recurrence rule in iCalendar RRULE format, e.g. 'FREQ=WEEKLY;BYDAY=MO' for weekly on Monday. Use with create_event or update_event."}
                 },
                 "required": ["action"]
             }
@@ -580,15 +569,14 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "manage_notes",
-            "description": "Manage notes and checklists (Google Keep-style): list, view, add, update, delete, toggle_item. Use list/search to find candidate notes, then view with the note id when you need the full body. IMPORTANT: For to-do lists / checklists, set note_type='checklist' and pass the items as the `checklist_items` array — do NOT serialize them into `content` as plain text. For freeform notes, use note_type='note' and put the body in `content`. `due_date` accepts natural language like 'tomorrow at 9am' (parsed in the user's timezone) and fires a notification — do not also create a calendar event for the same reminder.",
+            "description": "Manage notes and checklists (Google Keep-style): list, add, update, delete, toggle_item. IMPORTANT: For to-do lists / checklists, set note_type='checklist' and pass the items as the `checklist_items` array — do NOT serialize them into `content` as plain text. For freeform notes, use note_type='note' and put the body in `content`. `due_date` accepts natural language like 'tomorrow at 9am' (parsed in the user's timezone) and fires a notification — do not also create a calendar event for the same reminder.",
             "parameters": {
                 "type": "object",
                 "properties": {
                     "action": {"type": "string",
-                               "enum": ["list", "search", "view", "add", "update", "delete", "toggle_item"],
+                               "enum": ["list", "add", "update", "delete", "toggle_item"],
                                "description": "The action to perform"},
                     "id": {"type": "string", "description": "Note id (for update/delete/toggle_item); 8-char prefix is fine"},
-                    "query": {"type": "string", "description": "Search text for action='search'"},
                     "title": {"type": "string", "description": "Note title (for add/update)"},
                     "content": {"type": "string", "description": "Freeform body text. Use this for note_type='note'. Do NOT use this for checklists — pass `checklist_items` instead."},
                     "note_type": {"type": "string", "enum": ["note", "checklist"],
@@ -1035,7 +1023,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "manage_contact",
-            "description": "Create, update, delete, or list the user's CardDAV contacts. Use to save a new contact, update an existing one (email/phone/address), or remove one. Add does not require email: name + phone or name + address is valid. For update/delete you need the contact's uid — call action='list' first to find it. Writes go through the same dedupe + validation as the Contacts UI.",
+            "description": "Create, update, delete, or list the user's CardDAV contacts. Use to save a new contact, update an existing one (email/phone/address), or remove one. For update/delete you need the contact's uid — call action='list' first to find it. Writes go through the same dedupe + validation as the Contacts UI.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -1043,9 +1031,9 @@ FUNCTION_TOOL_SCHEMAS = [
                                "description": "list = show all contacts (with uids); add = create; update = edit by uid; delete = remove by uid."},
                     "uid": {"type": "string", "description": "Contact UID (required for update/delete; get it from action=list)."},
                     "name": {"type": "string", "description": "Contact's display name (for add/update)."},
-                    "email": {"type": "string", "description": "Single email address (convenience for add, or the primary email for update). Optional when phone or address is provided."},
+                    "email": {"type": "string", "description": "Single email address (convenience for add, or the primary email for update)."},
-                    "emails": {"type": "array", "items": {"type": "string"}, "description": "Full list of email addresses (first is primary)."},
+                    "emails": {"type": "array", "items": {"type": "string"}, "description": "Full list of email addresses (for update; first is primary)."},
-                    "phones": {"type": "array", "items": {"type": "string"}, "description": "Full list of phone numbers. Valid for add/update."},
+                    "phones": {"type": "array", "items": {"type": "string"}, "description": "Full list of phone numbers (for update)."},
                     "address": {"type": "string", "description": "Postal/mailing address as a single human-readable string."},
                 },
                 "required": ["action"]
@@ -1118,7 +1106,7 @@ FUNCTION_TOOL_SCHEMAS = [
         "type": "function",
         "function": {
             "name": "reply_to_email",
-            "description": "SEND a reply email immediately by UID. Do not use this when the user asks to write/draft/open/start a reply; use ui_control action=open_email_reply with body instead so the user can review. Only use when the user explicitly says to send now. Use the exact UID from the latest read_email/list_emails result; never invent UID 1. Automatically threads with In-Reply-To/References headers.",
+            "description": "SEND a reply email immediately by UID. Do not use this when the user asks to open/start a reply window or draft; use ui_control action=open_email_reply instead. For follow-up 'reply ...' requests where the user clearly wants to send now, use the exact UID from the latest read_email/list_emails result; never invent UID 1. Automatically threads with In-Reply-To/References headers.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -1222,113 +1210,38 @@ FUNCTION_TOOL_SCHEMAS = [
 # Converter: native function call -> ToolBlock
 # ---------------------------------------------------------------------------
 
-def _decode_loose_json_string(value: str) -> str:
-    """Decode common JSON string escapes without requiring inner quotes to be escaped."""
-    out = []
-    i = 0
-    while i < len(value):
-        ch = value[i]
-        if ch != "\\" or i + 1 >= len(value):
-            out.append(ch)
-            i += 1
-            continue
-        nxt = value[i + 1]
-        if nxt == "n":
-            out.append("\n")
-        elif nxt == "r":
-            out.append("\r")
-        elif nxt == "t":
-            out.append("\t")
-        elif nxt == "b":
-            out.append("\b")
-        elif nxt == "f":
-            out.append("\f")
-        elif nxt in ('"', "\\", "/"):
-            out.append(nxt)
-        elif nxt == "u" and i + 5 < len(value):
-            try:
-                out.append(chr(int(value[i + 2:i + 6], 16)))
-                i += 4
-            except ValueError:
-                out.append("\\" + nxt)
-        else:
-            out.append("\\" + nxt)
-        i += 2
-    return "".join(out)
-
-
-def _repair_document_function_args(tool_type: str, arguments: str) -> Optional[dict]:
-    """Salvage obvious malformed document tool args from local model wrappers.
-
-    The doc LoRA sometimes emits the right native tool call but puts raw quotes
-    inside the document text, making the surrounding JSON invalid. Treat that as
-    a wrapper parse failure, not a semantic tool-choice failure.
-    """
-    if tool_type != "update_document" or not isinstance(arguments, str):
-        return None
-    raw = arguments.strip()
-    if not raw.startswith("{") or not raw.endswith("}"):
-        return None
-    for key in ("content", "conten"):
-        marker = f'"{key}"'
-        key_pos = raw.find(marker)
-        if key_pos < 0:
-            continue
-        colon_pos = raw.find(":", key_pos + len(marker))
-        if colon_pos < 0:
-            continue
-        first_quote = raw.find('"', colon_pos + 1)
-        if first_quote < 0:
-            continue
-        close_brace = raw.rfind("}")
-        last_quote = raw.rfind('"', first_quote + 1, close_brace)
-        if last_quote <= first_quote:
-            continue
-        content = _decode_loose_json_string(raw[first_quote + 1:last_quote])
-        return {"content": content}
-    return None
-
-
 def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock]:
     """Convert a native function call into a ToolBlock for the existing execution pipeline."""
-    tool_type = _TOOL_NAME_MAP.get(name, name)
     try:
         if not arguments or (isinstance(arguments, str) and not arguments.strip()):
             args = {}
         else:
             args = json.loads(arguments) if isinstance(arguments, str) else arguments
     except (json.JSONDecodeError, TypeError):
-        args = _repair_document_function_args(tool_type, arguments)
+        logger.error(f"Failed to parse function call arguments for {name}: {arguments}")
-        if args is not None:
+        return None
-            logger.warning(f"Repaired malformed document function call arguments for {name}")
+
-        else:
+    tool_type = _TOOL_NAME_MAP.get(name, name)
-            logger.error(f"Failed to parse function call arguments for {name}: {arguments}")
+    _BUILTIN_EMAIL_TOOLS = {"list_email_accounts", "send_email", "list_emails", "read_email", "reply_to_email",
-            return None
+                            "archive_email", "delete_email", "mark_email_read", "bulk_email", "download_attachment"}
 
     # Some models emit valid JSON that isn't an object (e.g. a bare array
     # ["ls -la"], string, or number) as function arguments. Most local tools keep
     # the legacy empty-object coercion for stream robustness, but email MCP tools
     # must fail closed so a malformed call cannot read the default mailbox.
-    # Uses the shared BUILTIN_EMAIL_TOOLS (single source of truth) so the
-    # fail-closed set can't drift from the dispatch/blocklist sets.
     if not isinstance(args, dict):
-        if tool_type.startswith("mcp__email__") or name in BUILTIN_EMAIL_TOOLS:
+        if tool_type.startswith("mcp__email__") or name in _BUILTIN_EMAIL_TOOLS:
             logger.warning(f"Non-object email function call arguments for {name}: {args!r}; rejecting")
             return None
         logger.warning(f"Non-object function call arguments for {name}: {args!r}; treating as empty")
         args = {}
 
-    required_args = _REQUIRED_NATIVE_TOOL_ARGS.get(tool_type)
-    if required_args and not any(str(args.get(key) or "").strip() for key in required_args):
-        logger.warning(f"Rejecting empty required arguments for function call {name}: {args!r}")
-        return None
-
     # Allow MCP tools through (namespaced as mcp__serverid__toolname)
     if tool_type.startswith("mcp__"):
         content = json.dumps(args) if args else "{}"
         return ToolBlock(tool_type, content)
     # Email tools are implemented as MCP — route them to email
-    if name in BUILTIN_EMAIL_TOOLS:
+    if name in _BUILTIN_EMAIL_TOOLS:
         return ToolBlock(f"mcp__email__{name}", json.dumps(args) if args else "{}")
     if tool_type not in TOOL_TAGS:
         logger.warning(f"Unknown function call: {name}")
@@ -1430,20 +1343,15 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
     elif tool_type == "manage_memory":
         action = args.get("action", "")
         if action == "add":
-            text = args.get("text") or args.get("value") or args.get("content") or ""
+            content = "add\n" + args.get("text", "")
-            if not text and args.get("key"):
-                text = str(args.get("key") or "")
-            content = "add\n" + str(text)
             if args.get("category"):
                 content += "\n" + args["category"]
-            elif args.get("key"):
-                content += "\n" + str(args["key"])
         elif action == "edit":
             content = "edit\n" + args.get("memory_id", "") + "\n" + args.get("text", "")
         elif action == "delete":
             content = "delete\n" + args.get("memory_id", "")
         elif action == "search":
-            content = "search\n" + (args.get("text") or args.get("tex") or args.get("query") or "")
+            content = "search\n" + args.get("text", "")
         elif action == "list":
             content = "list"
             if args.get("category"):
@@ -1465,9 +1373,6 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock
             folder = args.get("folder") or value or "INBOX"
             mode = args.get("mode") or "reply"
             content = f"open_email_reply {uid} {folder} {mode}"
-            body = args.get("body") or args.get("extra") or args.get("content") or ""
-            if body:
-                content += f" {body}"
         elif action == "set_mode":
             content = f"set_mode {value or name}"
         elif action == "switch_model":

src/tool_security.py

@@ -8,36 +8,10 @@ from typing import Optional, Set
 logger = logging.getLogger(__name__)
 
 
-# Every tool exposed by the built-in email MCP server
-# (mcp_servers/email_server.py). Single source of truth: the fence tags
-# (TOOL_TAGS), bare-name dispatch (tool_execution), native-call mapping
-# (tool_schemas), and the non-admin blocklist below all derive from this set,
-# so a tool added to the email server can't become reachable under its bare
-# name without also being blocked for non-admins.
-BUILTIN_EMAIL_TOOLS = frozenset({
-    "list_email_accounts",
-    "list_emails",
-    "read_email",
-    "search_emails",
-    "send_email",
-    "reply_to_email",
-    "draft_email",
-    "draft_email_reply",
-    "ai_draft_email_reply",
-    "archive_email",
-    "delete_email",
-    "mark_email_read",
-    "bulk_email",
-    "download_attachment",
-})
-
-
 # Tools regular/public users must not execute directly. These either expose
 # server/runtime access, sensitive user data, external messaging, persistent
-# state changes, or generic loopback/integration surfaces. All email tools are
+# state changes, or generic loopback/integration surfaces.
-# included (SECURITY.md: email/MCP capabilities are privileged admin
+NON_ADMIN_BLOCKED_TOOLS = {
-# functionality).
-NON_ADMIN_BLOCKED_TOOLS = BUILTIN_EMAIL_TOOLS | {
     "bash",
     "python",
     "manage_bg_jobs",
@@ -60,6 +34,10 @@ NON_ADMIN_BLOCKED_TOOLS = BUILTIN_EMAIL_TOOLS | {
     "manage_settings",
     "api_call",
     "app_api",
+    "send_email",
+    "reply_to_email",
+    "list_emails",
+    "read_email",
     "resolve_contact",
     "manage_contact",
     "manage_calendar",
@@ -96,20 +74,8 @@ PLAN_MODE_READONLY_TOOLS = {
     "search_chats",
     "list_models",
     "list_sessions",
-    # Read-only email tools. list_email_accounts must be here because the
-    # bare/qualified alias gate in execute_tool_block works both ways: it has
-    # a native function schema, so plan mode's schema-derived bare denylist
-    # contains it — and without this allowlist entry that bare entry would
-    # also block the qualified mcp__email__list_email_accounts call that the
-    # MCP read-only filter deliberately allows.
-    "list_email_accounts",
     "list_emails",
     "read_email",
-    # Explicitly read-only rather than allowed-by-omission: this PR makes
-    # every BUILTIN_EMAIL_TOOLS name fence-taggable, so each one must be
-    # classified — see the plan-mode partition test in
-    # tests/test_email_registry_sync.py.
-    "search_emails",
     "list_served_models",
     "list_downloads",
     "list_cached_models",
@@ -143,14 +109,7 @@ _PLAN_MODE_KNOWN_MUTATORS = {
     "manage_webhooks", "manage_tokens", "manage_settings", "manage_contact",
     "manage_calendar", "api_call", "app_api", "ui_control",
     "send_email", "reply_to_email", "bulk_email", "delete_email",
-    "archive_email", "mark_email_read",
+    "archive_email", "mark_email_read", "download_model", "serve_model",
-    # The draft tools create documents and download_attachment writes to
-    # disk — mutating. They have no native schemas (yet), so without these
-    # static entries plan-mode safety for their bare fence tags would depend
-    # entirely on the MCP read-only inventory being present and current.
-    "draft_email", "draft_email_reply", "ai_draft_email_reply",
-    "download_attachment",
-    "download_model", "serve_model",
     "stop_served_model", "cancel_download", "adopt_served_model", "serve_preset",
     "generate_image", "edit_image", "trigger_research", "manage_research",
     # Shell is never read-only-safe; block it explicitly so it stays out of plan
@@ -192,28 +151,6 @@ def plan_mode_disabled_tools() -> Set[str]:
     return (all_names | _PLAN_MODE_KNOWN_MUTATORS) - PLAN_MODE_READONLY_TOOLS
 
 
-def email_tool_policy_names(tool_name: str) -> frozenset:
-    """All policy-equivalent spellings of a tool name.
-
-    A bare built-in email tool name and its MCP-qualified mcp__email__<name>
-    form dispatch to the same email server tool, but policy sources spell
-    them either way — plan mode and the MCP settings toggle write qualified
-    names into denylists, chat-level toggles write bare ones. Every gate must
-    match against the full alias set, or a call in one spelling slips past a
-    denylist entry written in the other. Non-email names alias only to
-    themselves.
-    """
-    if not isinstance(tool_name, str):
-        return frozenset((tool_name,))
-    if tool_name in BUILTIN_EMAIL_TOOLS:
-        return frozenset((tool_name, f"mcp__email__{tool_name}"))
-    if tool_name.startswith("mcp__email__"):
-        bare = tool_name[len("mcp__email__"):]
-        if bare in BUILTIN_EMAIL_TOOLS:
-            return frozenset((tool_name, bare))
-    return frozenset((tool_name,))
-
-
 def is_public_blocked_tool(tool_name: Optional[str]) -> bool:
     """Return True when a non-admin/public user must not execute this tool.
 

src/tool_utils.py

@@ -54,8 +54,6 @@ def _parse_tool_args(content):
     if isinstance(content, str):
         try:
             args = json.loads(content) if content.strip() else {}
-            if not isinstance(args, dict):
-                args = {}
         except (json.JSONDecodeError, TypeError) as e:
             raise ValueError(str(e))
     elif isinstance(content, dict):

src/tools/calendar.py

@@ -208,20 +208,11 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
         elif action == "list_events":
             try:
                 start_raw = _first_nonempty_arg(
-                    "start", "start_time", "start_date", "range_start", "from", "dtstart", "since"
+                    "start", "start_date", "range_start", "from", "dtstart", "since"
                 )
                 end_raw = _first_nonempty_arg(
-                    "end", "end_time", "end_date", "range_end", "to", "dtend", "until"
+                    "end", "end_date", "range_end", "to", "dtend", "until"
                 )
-                query_raw = args.get("query") or args.get("date_range") or args.get("range")
-                if query_raw and (not start_raw or not end_raw):
-                    return {
-                        "error": (
-                            "list_events needs explicit start/end ISO datetimes; "
-                            f"resolve the requested range ({query_raw!r}) and call manage_calendar again."
-                        ),
-                        "exit_code": 1,
-                    }
                 if start_raw:
                     start_dt = _parse_dt(start_raw)
                 else:
@@ -263,7 +254,6 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
                     "calendar_href": ev.calendar_id,
                     "event_type": ev.event_type or "",
                     "importance": ev.importance or "normal",
-                    "rrule": ev.rrule or "",
                 })
             if not events:
                 response_text = f"No events between {start_dt.date().isoformat()} and {end_dt.date().isoformat()}."
@@ -278,8 +268,6 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
                         line += f" #{ev['event_type']}"
                     if ev.get("importance") and ev["importance"] != "normal":
                         line += f" !{ev['importance']}"
-                    if ev.get("rrule"):
-                        line += f" repeats({ev['rrule']})"
                     if ev.get("location"):
                         line += f" @ {ev['location']}"
                     if ev.get("calendar"):
@@ -492,10 +480,6 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
                 ev.event_type = _tag or None
             if args.get("importance") is not None:
                 ev.importance = args["importance"]
-            if args.get("rrule") is not None:
-                ev.rrule = args.get("rrule") or ""
-            elif str(args.get("repeat") or "").strip().lower() in {"none", "no", "off", "false", "single"}:
-                ev.rrule = ""
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
             if is_caldav:
                 ev.caldav_sync_pending = "update"

src/tools/contacts.py

@@ -108,28 +108,16 @@ async def do_manage_contact(content: str, owner: Optional[str] = None) -> Dict:
 
         if action == "add":
             email = (args.get("email") or "").strip()
-            phones = [str(p or "").strip() for p in (args.get("phones") or []) if str(p or "").strip()]
+            if not email:
-            phone = (args.get("phone") or "").strip()
+                return {"error": "email is required for add", "exit_code": 1}
-            if phone and phone not in phones:
+            name = (args.get("name") or "").strip() or email.split("@")[0]
-                phones.insert(0, phone)
+            # Dedupe by email (same as the /add route).
-            address = (args.get("address") or "").strip()
-            name = (args.get("name") or "").strip()
-            if not name and email:
-                name = email.split("@")[0]
-            if not name and not email and not phones and not address:
-                return {"error": "name plus email, phone, or address is required for add", "exit_code": 1}
-            if not name:
-                name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
-            # Dedupe by email or phone (same as the /add route).
             existing = await asyncio.to_thread(cc._fetch_contacts)
             for c in existing:
-                if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
+                if email.lower() in [e.lower() for e in c.get("emails", [])]:
                     return {"output": f"{email} is already a contact ({c.get('name','')}).", "exit_code": 0}
-                if phones and any(p in (c.get("phones") or []) for p in phones):
+            ok = await asyncio.to_thread(cc._create_contact, name, email)
-                    return {"output": f"{phones[0]} is already a contact ({c.get('name','')}).", "exit_code": 0}
+            return {"output": f"{'Added' if ok else 'Failed to add'} {name} <{email}>.", "exit_code": 0 if ok else 1}
-            ok = await asyncio.to_thread(cc._create_contact, name, email, address, phones)
-            detail = email or ", ".join(phones) or address
-            return {"output": f"{'Added' if ok else 'Failed to add'} {name} ({detail}).", "exit_code": 0 if ok else 1}
 
         if action in ("update", "edit"):
             uid = (args.get("uid") or "").strip()
@@ -141,12 +129,11 @@ async def do_manage_contact(content: str, owner: Optional[str] = None) -> Dict:
                 emails = [args["email"]]
             emails = [e.strip() for e in (emails or []) if e and e.strip()]
             phones = [p.strip() for p in (args.get("phones") or []) if p and p.strip()]
-            address = (args.get("address") or "").strip()
+            if not name and not emails:
-            if not name and not emails and not phones and not address:
+                return {"error": "Provide a name or emails to update", "exit_code": 1}
-                return {"error": "Provide a name, emails, phones, or address to update", "exit_code": 1}
             if not name and emails:
                 name = emails[0].split("@")[0]
-            ok = await asyncio.to_thread(cc._update_contact, uid, name, emails, phones, address)
+            ok = await asyncio.to_thread(cc._update_contact, uid, name, emails, phones)
             return {"output": "Contact updated." if ok else "Update failed.", "exit_code": 0 if ok else 1}
 
         if action == "delete":

src/tools/cookbook.py

@@ -315,7 +315,6 @@ async def _cookbook_register_task(
 _MODEL_PROCESS_PATTERNS = [
     ("vLLM",            ["vllm.entrypoints", "vllm serve", "/vllm/", "vllm-openai"]),
     ("SGLang",          ["sglang.launch_server", "sglang/launch_server"]),
-    ("MLX",             ["mlx_lm.server", "mlx-lm"]),
     ("llama.cpp",       ["llama-server", "llama_cpp_server", "llamacppserver"]),
     ("Ollama",          ["ollama serve", "ollama runner", "/ollama "]),
     ("ComfyUI",         ["comfyui/main.py", "/ComfyUI/main.py", "ComfyUI"]),
@@ -591,7 +590,7 @@ async def do_serve_model(content: str, owner: Optional[str] = None) -> Dict:
         hint = ""
         if isinstance(err_msg, str) and "cmd" in err_msg.lower():
             hint = (" — the cmd must START with an allowlisted binary "
-                    "(vllm, python3, llama-server, ollama, sglang, mlx_lm, lmdeploy, node, npx). "
+                    "(vllm, python3, llama-server, ollama, sglang, lmdeploy, node, npx). "
                     "Do NOT prefix with `cd …`, `source …`, or chain with `&&`. "
                     "env_prefix (e.g. `source ~/qwen35-env/bin/activate`) is added "
                     "automatically from the host's saved venv settings.")
@@ -636,7 +635,7 @@ async def do_list_served_models(content: str, owner: Optional[str] = None) -> Di
 
     if not merged:
         return {
-            "output": "No model servers currently running (cookbook task tracker empty; /proc scan found no vLLM / sglang / MLX / llama.cpp / Ollama / ComfyUI / A1111 / Fooocus / InvokeAI / TGI / Aphrodite / Triton / Diffusers processes).",
+            "output": "No model servers currently running (cookbook task tracker empty; /proc scan found no vLLM / sglang / llama.cpp / Ollama / ComfyUI / A1111 / Fooocus / InvokeAI / TGI / Aphrodite / Triton / Diffusers processes).",
             "exit_code": 0,
         }