Jake/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-07-16 13:08:02 +00:00
Code Packages Projects Releases Activity

Compare commits

base: Jake/odysseus:dev
Branches Tags
Jake/odysseus:dev Jake/odysseus:main Jake/odysseus:dependabot/pip/python-258f0448df Jake/odysseus:dependabot/github_actions/actions-5b4a7f4463 Jake/odysseus:fix/security-gallery-endpoint-hardening
..
compare: Jake/odysseus:bd0c67b6d3db7d9a5de2e62b69369e134d6507e2
Branches Tags
Jake/odysseus:dev Jake/odysseus:main Jake/odysseus:dependabot/pip/python-258f0448df Jake/odysseus:dependabot/github_actions/actions-5b4a7f4463 Jake/odysseus:fix/security-gallery-endpoint-hardening

2 Commits
dev .. bd0c67b6d3

Author SHA1 Message Date
Alexandre Teixeira bd0c67b6d3 fix(agent): preserve loop guard stream behavior 2026-06-15 17:17:16 +01:00
Alexandre Teixeira ff5bcd9864 fix(agent): surface early loop-guard stops 2026-06-15 17:07:15 +01:00
548 changed files with 17952 additions and 81850 deletions
Expand all files Collapse all files
.dockerignore
-4
View File
@@ -15,10 +15,6 @@ build/
# at runtime — never baked into the image. Mirrored in .gitignore.
secrets.env
secrets.env.*
secrets.env~
.secrets.env.swp
.secrets.env.swo
**/#secrets.env#
!secrets.env.example
/data/
/logs/
.env.example
-20
View File
@@ -169,26 +169,6 @@ SEARXNG_INSTANCE=http://localhost:8080
# ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400 # speech-to-text audio (25 MB)
# ODYSSEUS_ICS_MAX_BYTES=10485760 # calendar .ics import (10 MB)
# ============================================================
# Host Docker access (explicit opt-in)
# ============================================================
# Default Docker Compose does not mount /var/run/docker.sock. Existing
# Ollama, vLLM, and other OpenAI-compatible endpoints remain usable without it.
#
# Enable this only for intentional Cookbook/local Docker-daemon management.
# Raw socket access is high-trust and can grant broad control over the host
# Docker daemon. Set DOCKER_GID to the host docker group's numeric GID.
# Put these values in .env, or export them before running docker compose.
# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
# DOCKER_GID=963
# docker/host-docker.yml sets this inside the container. Keep it paired
# with the socket overlay; setting it alone is not sufficient.
# ODYSSEUS_ENABLE_HOST_DOCKER=true
#
# Host Docker access can be combined with one GPU overlay:
# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
# ============================================================
# GPU support (Docker Compose)
# ============================================================
.github/scripts/focused_test_guidance.py
-124
View File
@@ -1,124 +0,0 @@
#!/usr/bin/env python3
"""Report focused pytest guidance for changed paths under tests/."""
from __future__ import annotations
import argparse
import os
import shlex
import subprocess
import sys
from collections.abc import Iterable
from pathlib import PurePosixPath
def parse_paths(raw_paths: bytes) -> list[str]:
"""Decode the NUL-delimited output of ``git diff --name-only -z``."""
return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path]
def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]:
"""Return changed ``tests/`` paths using GitHub PR three-dot semantics.
GitHub PR changed files are based on the merge base and the PR head, not a
direct endpoint diff between the current base branch tip and the PR head.
Using the direct endpoint diff can include files changed only on the base
branch when the PR branch is stale.
"""
merge_base = subprocess.check_output(
["git", "merge-base", base_sha, head_sha],
stderr=subprocess.DEVNULL,
).strip()
raw_paths = subprocess.check_output(
[
"git",
"diff",
"--name-only",
"--diff-filter=ACMRT",
"-z",
os.fsdecode(merge_base),
head_sha,
"--",
"tests/",
],
)
return parse_paths(raw_paths)
def select_test_paths(paths: Iterable[str]) -> list[str]:
"""Return unique, repository-relative paths contained by tests/."""
selected: set[str] = set()
for raw_path in paths:
path = PurePosixPath(raw_path)
if path.is_absolute() or ".." in path.parts:
continue
parts = tuple(part for part in path.parts if part != ".")
if len(parts) >= 2 and parts[0] == "tests":
selected.add(PurePosixPath(*parts).as_posix())
return sorted(selected)
def is_pytest_file(path: str) -> bool:
"""Return whether a changed path follows this repository's pytest naming."""
name = PurePosixPath(path).name
return name.endswith(".py") and (
name.startswith("test_") or name.endswith("_test.py")
)
def pytest_command(paths: Iterable[str]) -> str:
"""Build a copyable pytest command for changed runnable test files."""
command = ["python3", "-m", "pytest", "-q", *paths]
return shlex.join(command)
def format_report(paths: Iterable[str]) -> str:
"""Format focused guidance for CI logs and the workflow summary."""
changed_paths = select_test_paths(paths)
runnable_paths = [path for path in changed_paths if is_pytest_file(path)]
lines = ["## Focused test guidance (report-only)", ""]
if not changed_paths:
lines.append("No changed paths under `tests/`.")
else:
lines.extend(["Changed paths under `tests/`:", ""])
lines.extend(f"- `{path}`" for path in changed_paths)
lines.extend(["", "Suggested focused validation:", ""])
if runnable_paths:
lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```")
else:
lines.append("No directly runnable pytest files changed.")
lines.extend(
[
"",
"This guidance does not infer tests from source changes. "
"Existing blocking CI remains the source of truth.",
]
)
return "\n".join(lines)
def _parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Report focused pytest guidance for changed tests/ paths.",
)
parser.add_argument("--base-sha", help="Pull request base commit SHA.")
parser.add_argument("--head-sha", help="Pull request head commit SHA.")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = _parse_args(sys.argv[1:] if argv is None else argv)
if bool(args.base_sha) != bool(args.head_sha):
raise SystemExit("--base-sha and --head-sha must be provided together")
if args.base_sha and args.head_sha:
paths = changed_paths_from_merge_base(args.base_sha, args.head_sha)
else:
paths = parse_paths(sys.stdin.buffer.read())
print(format_report(paths))
return 0
if __name__ == "__main__":
raise SystemExit(main())
.github/workflows/ci.yml
+3 -57
View File
@@ -15,65 +15,11 @@ concurrency:
cancel-in-progress: true
jobs:
focused-test-guidance:
name: Focused test guidance (report-only)
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Report changed test paths
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
report_file="$RUNNER_TEMP/focused-test-guidance.md"
publish_report() {
cat "$report_file"
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true
fi
return 0
}
report_unavailable() {
{
printf '%s\n\n' '## Focused test guidance unavailable (report-only)'
printf '%s\n\n' "$1"
printf '%s\n' 'Existing blocking CI remains the source of truth.'
} > "$report_file"
publish_report
exit 0
}
if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
report_unavailable "Pull request base/head metadata is missing."
fi
if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
report_unavailable "The pull request base commit is unavailable locally."
fi
if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then
report_unavailable "The pull request head commit is unavailable locally."
fi
if ! python3 .github/scripts/focused_test_guidance.py \
--base-sha "$BASE_SHA" \
--head-sha "$HEAD_SHA" > "$report_file"; then
report_unavailable "The focused test guidance helper could not produce a report."
fi
publish_report
python-syntax:
name: Python syntax (compileall)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
@@ -86,7 +32,7 @@ jobs:
name: JS syntax (node --check)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
@@ -108,7 +54,7 @@ jobs:
# ROADMAP "fresh install smoke tests" item; make this required once green.
continue-on-error: true
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 0
persist-credentials: false
.github/workflows/container-scan.yml
+1 -1
View File
@@ -37,7 +37,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
.github/workflows/container-trivy.yml
+2 -2
View File
@@ -52,7 +52,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
@@ -93,7 +93,7 @@ jobs:
security-events: write # upload SARIF to the Security tab
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
.github/workflows/dependency-review.yml
+2 -2
View File
@@ -36,7 +36,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
@@ -55,7 +55,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
.github/workflows/docker-publish.yml
+2 -2
View File
@@ -45,7 +45,7 @@ jobs:
arch: arm64
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- name: Set up Buildx
@@ -86,7 +86,7 @@ jobs:
contents: read
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- name: Read APP_VERSION + short sha
.github/workflows/issue-description-check.yml
+1 -1
View File
@@ -14,7 +14,7 @@ jobs:
# Skip bots (Dependabot, release-drafter, etc.)
if: ${{ github.event.issue.user.type != 'Bot' }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
sparse-checkout: .github/scripts
persist-credentials: false
.github/workflows/pr-description-check.yml
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
# Skip bots: they open PRs programmatically and have their own process.
if: github.event.pull_request.user.type != 'Bot'
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.base_ref }}
sparse-checkout: .github/scripts
.github/workflows/secret-scan.yml
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
# Full history so a secret committed in an earlier commit (and later
# deleted) is still caught -- deletion does not remove it from Git.
.github/workflows/workflow-security.yml
+2 -2
View File
@@ -36,7 +36,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
@@ -61,7 +61,7 @@ jobs:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
ACKNOWLEDGMENTS.md
-1
View File
@@ -86,7 +86,6 @@ Bundled in `static/fonts/`:
| [Fira Code](https://github.com/tonsky/FiraCode) | SIL Open Font License 1.1 | Nikita Prokopov & contributors |
| [Inter](https://github.com/rsms/inter) | SIL Open Font License 1.1 | Rasmus Andersson |
| [GohuFont](https://font.gohu.org/) (`fonts/custom/GohuFont.ttf`) | WTFPL | Hugo Chargois |
| [OpenDyslexic](https://opendyslexic.org/) (`fonts/OpenDyslexic-{Regular,Bold}.woff2`) | SIL Open Font License 1.1 ([`licenses/OpenDyslexic-OFL.txt`](licenses/OpenDyslexic-OFL.txt)) | Abbie Gonzalez |
## Python dependencies
CONTRIBUTING.md
+1 -1
View File
@@ -37,7 +37,7 @@ Manual development uses Python 3.11+:
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
python -m uvicorn app:app --host 127.0.0.1 --port 7000
python -m uvicorn app:app --host 0.0.0.0 --port 7000
```
Windows is not actively tested. Docker on Linux or a Linux/macOS manual install is the safer path for now.
Dockerfile
-61
View File
@@ -1,14 +1,3 @@
# ---- builder: patch + build wheels for Real-ESRGAN's broken-on-3.14 deps ----
# basicsr/gfpgan/facexlib read their version via exec()+locals()['__version__'],
# which raises KeyError on Python 3.13+ (PEP 667). Build patched wheels here so
# the final image / Cookbook never has to compile the broken sdists. See
# docker/build-realesrgan-wheels.sh for the full rationale.
FROM python:3.14-slim AS realesrgan-wheels
RUN apt-get update && apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh
RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels
FROM python:3.14-slim
# System deps. tmux is required by Cookbook for background downloads/serves.
@@ -29,44 +18,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
tmux \
openssh-client \
gosu \
libgl1 \
libglib2.0-0t64 \
libxcb1 \
libmagic1 \
&& rm -rf /var/lib/apt/lists/*
# libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1,
# libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The
# slim base omits them, so the Cookbook "install realesrgan" path imports cv2
# and dies with `libxcb.so.1: cannot open shared object file` despite a clean
# pip install. Using full opencv-python (not -headless) because basicsr/gfpgan/
# facexlib/realesrgan all depend on the `opencv-python` distribution by name.
#
# libmagic1 is the shared lib (libmagic.so.1) that python-magic dlopens for
# content-based MIME sniffing in src/upload_handler.py. We install both here
# (libmagic1 + the python-magic wrapper, below) rather than in requirements.txt
# because python-magic resolves libmagic at import time: where the lib is
# absent the import can block or raise, so keeping it image-only avoids
# regressing pip/venv installs on hosts without libmagic. Debian always has the
# lib here, so the import is instant and detection actually works.
# Docker CLI (client only — daemon stays on the host via the
# /var/run/docker.sock mount). The Debian `docker.io` package ships
# dockerd but not the client binary on slim, so grab the static client
# tarball from download.docker.com instead.
ARG DOCKER_CLI_VERSION=27.5.1
RUN ARCH="$(dpkg --print-architecture)" \
&& case "$ARCH" in \
amd64) DARCH=x86_64 ;; \
arm64) DARCH=aarch64 ;; \
*) echo "unsupported arch $ARCH"; exit 1 ;; \
esac \
&& curl -fsSL "https://download.docker.com/linux/static/stable/${DARCH}/docker-${DOCKER_CLI_VERSION}.tgz" \
-o /tmp/docker.tgz \
&& tar -xzf /tmp/docker.tgz -C /tmp \
&& install -m 0755 /tmp/docker/docker /usr/local/bin/docker \
&& rm -rf /tmp/docker /tmp/docker.tgz
WORKDIR /app
# Install Python deps first (layer cache). Optional extras (PyMuPDF AGPL, etc.)
@@ -76,20 +29,6 @@ COPY requirements.txt requirements-optional.txt ./
RUN pip install --no-cache-dir -r requirements.txt \
&& if [ "$INSTALL_OPTIONAL" = "true" ]; then pip install --no-cache-dir -r requirements-optional.txt; fi
# python-magic powers content-based MIME sniffing in src/upload_handler.py.
# Image-only (not in requirements.txt) because it needs the libmagic1 system
# lib installed above; see the apt note near the top of this stage.
RUN pip install --no-cache-dir python-magic==0.4.27
# Pre-install the patched basicsr/gfpgan/facexlib wheels built in the
# realesrgan-wheels stage (--no-deps keeps the image lean — torch & friends are
# pulled only when realesrgan is actually installed). With these dists already
# satisfied, the Cookbook's plain `pip install realesrgan` resolves them from
# wheels instead of rebuilding the sdists that fail on Python 3.14.
COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/
RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \
&& rm -rf /tmp/odysseus-wheels
# Copy app code
COPY . .
Odysseus.spec
-45
View File
@@ -1,45 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
a = Analysis(
['launcher.py'],
pathex=[],
binaries=[],
datas=[('static', 'static'), ('scripts', 'scripts'), ('mcp_servers', 'mcp_servers'), ('services/hwfit/data', 'services/hwfit/data'), ('config', 'config'), ('.env.example', '.env.example')],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=True,
name='Odysseus',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
icon=['static\\icon.ico'],
)
coll = COLLECT(
exe,
a.binaries,
a.datas,
strip=False,
upx=True,
upx_exclude=[],
name='Odysseus',
)
README.md
+2 -2
View File
@@ -1,5 +1,5 @@
<p align="center">
<img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238">
<img src="docs/odysseus-wordmark.png" alt="Odysseus" width="280">
</p>
<p align="center">
@@ -18,7 +18,7 @@
</p>
<p align="center">
<img src="docs/odysseus-browser.jpg" alt="Odysseus interface">
<img src="docs/odysseus.jpg" alt="Odysseus interface">
</p>
---
ROADMAP.md
+1
View File
@@ -12,6 +12,7 @@ the codebase, you are probably right to stay away.
and WSL all need coverage.
- Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden.
- Self-host troubleshooting cookbook. Document the weird 30-second fixes that otherwise become 30-minute searches: Dovecot cleartext auth for local stacks, ntfy Android Instant Delivery for non-ntfy.sh servers, clipboard limits on plain-HTTP Tailscale URLs, Radicale collection URLs, and similar traps.
- Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments.
- Cookbook SGLang support across platforms. Make sure SGLang setup/serve works
predictably on Linux, Windows/WSL, macOS where possible, Docker, and common
app.py
+87 -195
View File
@@ -1,18 +1,6 @@
# app.py — slim orchestrator
import mimetypes
import os
import sys
import asyncio
import time
# On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
# When started via `python -m uvicorn` from a terminal, uvicorn sets this
# automatically. But the VS Code debugger (and other non-uvicorn entrypoints)
# use the default SelectorEventLoop, which raises NotImplementedError on any
# subprocess call. Force ProactorEventLoop here so the right loop is always
# used, regardless of how the process is launched.
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
def register_static_mime_types() -> None:
@@ -50,12 +38,12 @@ load_dotenv(encoding="utf-8-sig")
import asyncio
import logging
import secrets
from datetime import datetime, timezone
from datetime import datetime
from typing import Dict
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import JSONResponse, FileResponse
from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.middleware.base import BaseHTTPMiddleware
@@ -76,7 +64,7 @@ from core.exceptions import (
import bcrypt as _bcrypt
from src.app_helpers import abs_join, serve_html_with_nonce
from src.app_helpers import abs_join
from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_image_path
from starlette.responses import RedirectResponse
@@ -125,13 +113,12 @@ app = FastAPI(
)
# ========= CORS =========
CORS_ALLOW_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"]
allowed_origins = os.getenv("ALLOWED_ORIGINS", "http://localhost,http://127.0.0.1").split(",")
app.add_middleware(
CORSMiddleware,
allow_origins=allowed_origins,
allow_credentials=True,
allow_methods=CORS_ALLOW_METHODS,
allow_methods=["GET", "POST", "PUT", "DELETE"],
allow_headers=[
"Accept",
"Authorization",
@@ -198,50 +185,7 @@ class _RequestTimeoutMiddleware(_BaseHTTPMiddleware):
)
class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
from src.interactive_gate import should_track_interactive_request, track_interactive_request
path = request.url.path or ""
if not should_track_interactive_request(path, request.method):
return await call_next(request)
async def _stop_background():
try:
await task_scheduler.stop_background_tasks_for_foreground(reason=f"foreground request {request.method} {path}")
except Exception:
logging.getLogger("app.foreground_gate").debug("foreground task stop failed", exc_info=True)
asyncio.create_task(_stop_background())
async with track_interactive_request(path, request.method):
return await call_next(request)
class _SlowRequestLogMiddleware(_BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
start = time.perf_counter()
status = 500
try:
response = await call_next(request)
status = getattr(response, "status_code", 0) or 0
return response
finally:
elapsed = time.perf_counter() - start
try:
threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75")
except Exception:
threshold = 0.75
if elapsed >= threshold:
logging.getLogger("app.slow_request").warning(
"slow_request method=%s path=%s status=%s elapsed=%.3fs",
request.method,
request.url.path,
status,
elapsed,
)
app.add_middleware(_RequestTimeoutMiddleware)
app.add_middleware(_InteractiveActivityMiddleware)
app.add_middleware(_SlowRequestLogMiddleware)
# ========= AUTH =========
from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
@@ -372,7 +316,7 @@ if AUTH_ENABLED:
# (no admin cookie available in that context). Restricted to
# loopback clients + matching token to keep it locked down.
try:
from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT, INTERNAL_TOOL_USER
from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT
_hdr = request.headers.get(INTERNAL_TOOL_HEADER)
if _hdr and secrets.compare_digest(_hdr, _ITT) and _is_trusted_loopback(request):
# Impersonation: when the agent's loopback call sets
@@ -384,11 +328,11 @@ if AUTH_ENABLED:
if _impersonate and _impersonate in getattr(_auth_mgr, "users", {}):
request.state.current_user = _impersonate
else:
request.state.current_user = INTERNAL_TOOL_USER
request.state.current_user = "internal-tool"
request.state.api_token = False
return await call_next(request)
except Exception as _e:
logger.warning("Internal tool auth header check failed", exc_info=_e)
except Exception:
pass
# Allow DIRECT localhost requests (internal service calls from
# heartbeats etc.). Tunnel/proxy-forwarded requests are excluded by
# _is_trusted_loopback so LOCALHOST_BYPASS can't be abused over a
@@ -441,10 +385,11 @@ if AUTH_ENABLED:
_db.close()
try:
await _asyncio.to_thread(_do)
except Exception as _e:
logger.debug("Failed to update token last_used_at", exc_info=_e)
except Exception:
pass
_asyncio.create_task(_touch_last_used(matched_id))
# Keep bearer-token callers out of normal cookie/user
# routes. API-aware routes can read api_token_owner.
request.state.current_user = "api"
request.state.api_token = True
request.state.api_token_id = matched_id
@@ -493,7 +438,7 @@ class _RevalidatingStatic(StaticFiles):
return resp
app.mount("/static", _RevalidatingStatic(directory=STATIC_DIR), name="static")
app.mount("/static", _RevalidatingStatic(directory="static"), name="static")
# ========= GENERATED IMAGES =========
@app.get("/api/generated-image/{filename}")
@@ -519,8 +464,8 @@ async def serve_generated_image(filename: str, request: Request):
_db.close()
except HTTPException:
raise
except Exception as _e:
logger.warning("Image ownership verification failed for %r", filename, exc_info=_e)
except Exception:
pass
ext = filename.rsplit('.', 1)[-1].lower()
mime = {
"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg",
@@ -583,7 +528,6 @@ memory_vector = components.get("memory_vector")
upload_handler = components["upload_handler"]
app.state.upload_handler = upload_handler
personal_docs_mgr = components["personal_docs_manager"]
app.state.personal_docs_manager = personal_docs_mgr
api_key_manager = components["api_key_manager"]
preset_manager = components["preset_manager"]
chat_processor = components["chat_processor"]
@@ -627,20 +571,6 @@ webhook_manager = WebhookManager(api_key_manager=api_key_manager)
auth_router = setup_auth_routes(auth_manager)
app.include_router(auth_router)
@app.post("/api/activity/heartbeat")
async def activity_heartbeat():
from src.interactive_gate import mark_browser_activity
await mark_browser_activity()
async def _stop_background():
try:
await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
except Exception:
logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
asyncio.create_task(_stop_background())
return {"ok": True}
# Uploads
from routes.upload_routes import setup_upload_routes
upload_router, upload_cleanup_func = setup_upload_routes(upload_handler)
@@ -655,19 +585,14 @@ app.include_router(setup_emoji_routes())
# Sessions
from routes.session_routes import setup_session_routes
session_config = {"REQUEST_TIMEOUT": REQUEST_TIMEOUT, "OPENAI_API_KEY": OPENAI_API_KEY, "SESSIONS_FILE": SESSIONS_FILE}
app.include_router(setup_session_routes(
session_manager,
session_config,
webhook_manager=webhook_manager,
upload_handler=upload_handler,
))
app.include_router(setup_session_routes(session_manager, session_config, webhook_manager=webhook_manager))
# Admin Danger Zone wipes (Settings → System → Danger Zone)
from routes.admin_wipe_routes import setup_admin_wipe_routes
app.include_router(setup_admin_wipe_routes(session_manager))
# Memory
from routes.memory.memory_routes import setup_memory_routes
from routes.memory_routes import setup_memory_routes
memory_router = setup_memory_routes(memory_manager, session_manager, memory_vector=memory_vector)
app.include_router(memory_router)
from routes.skills_routes import setup_skills_routes
@@ -684,12 +609,12 @@ app.include_router(setup_chat_routes(
))
# Research (background deep-research tasks)
from routes.research.research_routes import setup_research_routes
from routes.research_routes import setup_research_routes
app.include_router(setup_research_routes(research_handler, session_manager=session_manager))
# History
from routes.history.history_routes import setup_history_routes
app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler))
from routes.history_routes import setup_history_routes
app.include_router(setup_history_routes(session_manager))
# Search
from routes.search_routes import setup_search_routes
@@ -748,7 +673,7 @@ from routes.signature_routes import setup_signature_routes
app.include_router(setup_signature_routes())
# Gallery (image library)
from routes.gallery.gallery_routes import setup_gallery_routes
from routes.gallery_routes import setup_gallery_routes
app.include_router(setup_gallery_routes())
# Persisted image-editor drafts (server-backed projects)
@@ -768,7 +693,7 @@ app.include_router(setup_assistant_routes(task_scheduler))
# Calendar (CalDAV)
from routes.calendar_routes import setup_calendar_routes
calendar_router = setup_calendar_routes(upload_handler=upload_handler)
calendar_router = setup_calendar_routes()
app.include_router(calendar_router)
# Shell (user-facing command execution)
@@ -831,7 +756,7 @@ logger.info("Webhook & API token routes initialized")
# Notes (Google Keep-style notes/todos)
from routes.note_routes import setup_note_routes
app.include_router(setup_note_routes(task_scheduler, upload_handler=upload_handler))
app.include_router(setup_note_routes(task_scheduler))
# Email
from routes.email_routes import setup_email_routes
@@ -856,7 +781,7 @@ from routes.vault_routes import setup_vault_routes
app.include_router(setup_vault_routes())
# Contacts (CardDAV)
from routes.contacts.contacts_routes import setup_contacts_routes
from routes.contacts_routes import setup_contacts_routes
app.include_router(setup_contacts_routes())
from companion import setup_companion_routes
@@ -864,17 +789,23 @@ app.include_router(setup_companion_routes())
# ========= ROUTES (kept in app.py) =========
def _serve_html_with_nonce(request: Request, file_path: str) -> HTMLResponse:
"""Read an HTML file and inject the CSP nonce into inline <script> tags."""
with open(file_path, "r", encoding="utf-8") as f:
html = f.read()
nonce = getattr(request.state, "csp_nonce", "")
html = html.replace("{{CSP_NONCE}}", nonce)
return HTMLResponse(html)
@app.get("/")
async def serve_index(request: Request):
static_path = abs_join(BASE_DIR, "static/index.html")
if os.path.exists(static_path):
return serve_html_with_nonce(request, static_path)
# No static bundle — fall back to a root-level index.html if one is shipped.
# If neither exists, serve_html_with_nonce logs it and returns a generic 500:
# a missing index.html is a broken deployment (server fault), not a client
# "not found". This keeps the app-shell route consistent with the other
# bundled-template routes instead of mislabelling the fault as a 404.
return serve_html_with_nonce(request, abs_join(BASE_DIR, "index.html"))
return _serve_html_with_nonce(request, static_path)
root_path = abs_join(BASE_DIR, "index.html")
if os.path.exists(root_path):
return _serve_html_with_nonce(request, root_path)
raise HTTPException(404, "index.html not found")
@app.get("/notes")
async def serve_notes(request: Request):
@@ -915,13 +846,13 @@ async def serve_library(request: Request):
@app.get("/backgrounds")
async def serve_backgrounds(request: Request):
"""Sandbox page for prototyping background effects. No auth required."""
return serve_html_with_nonce(request, abs_join(BASE_DIR, "static/backgrounds.html"))
return _serve_html_with_nonce(request, abs_join(BASE_DIR, "static/backgrounds.html"))
@app.get("/login")
async def serve_login(request: Request):
if not AUTH_ENABLED:
return RedirectResponse(url="/", status_code=302)
return serve_html_with_nonce(request, abs_join(BASE_DIR, "static/login.html"))
return _serve_html_with_nonce(request, abs_join(BASE_DIR, "static/login.html"))
@app.get("/api/version")
async def get_version():
@@ -930,35 +861,7 @@ async def get_version():
@app.get("/api/health")
async def health_check() -> Dict[str, str]:
return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()}
@app.post("/api/client-perf")
async def client_perf(request: Request):
"""Low-volume frontend timing reports for stalls that happen before SSE logs."""
try:
data = await request.json()
except Exception:
data = {}
try:
kind = str(data.get("type") or "client").replace("\n", " ")[:80]
total_ms = float(data.get("total_ms") or 0)
stages = data.get("stages") if isinstance(data.get("stages"), list) else []
stage_txt = " ".join(
f"{str(s.get('name') or '')[:40]}={float(s.get('delta_ms') or 0):.0f}ms"
for s in stages[:20]
if isinstance(s, dict)
)
extra = str(data.get("extra") or "").replace("\n", " ")[:200]
logging.getLogger("app.client_perf").warning(
"client_perf type=%s total=%.0fms %s%s",
kind,
total_ms,
stage_txt,
f" extra={extra}" if extra else "",
)
except Exception:
logging.getLogger("app.client_perf").debug("client_perf log failed", exc_info=True)
return {"ok": True}
return {"status": "healthy", "timestamp": datetime.utcnow().isoformat()}
@app.get("/api/ready")
async def readiness_check() -> JSONResponse:
@@ -1048,7 +951,7 @@ async def _startup_event():
except BaseException as e:
logger.warning(f"Built-in MCP registration failed (non-critical): {type(e).__name__}: {e}")
try:
await mcp_manager.connect_all_enabled()
await asyncio.wait_for(mcp_manager.connect_all_enabled(), timeout=20)
except asyncio.TimeoutError:
logger.warning("User MCP startup timed out (non-critical)")
except BaseException as e:
@@ -1056,59 +959,57 @@ async def _startup_event():
_startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
# Startup warmups are opt-in. They make later requests a little warmer, but
# they also compete with the first seconds of real UI use on slow or busy
# machines. Default to clear/idle startup and let requests warm what they use.
_startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
if _startup_warmups_enabled:
async def _warmup_tool_index():
try:
from src.tool_index import get_tool_index
idx = await asyncio.to_thread(get_tool_index)
if idx:
await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
logger.info("[startup] Tool index pre-warmed")
except Exception as e:
logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
# Pre-warm the RAG tool index off the request path. Loading the local
# embedding model + opening ChromaDB + indexing the built-in tools is a
# one-time ~1-3s cost that otherwise lands on the user's FIRST message
# (showing up as a big `tool_selection` time). Doing it here makes the
# first turn as fast as subsequent ones (warm embed ≈ a few ms).
async def _warmup_tool_index():
try:
from src.tool_index import get_tool_index
idx = await asyncio.to_thread(get_tool_index)
if idx:
await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
logger.info("[startup] Tool index pre-warmed")
except Exception as e:
logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
_startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
async def _warmup_endpoints():
try:
import httpx
urls = (
await asyncio.to_thread(model_discovery.warmup_ping_urls)
if model_discovery else []
)
for url in urls:
try:
async with httpx.AsyncClient(timeout=5.0) as client:
await client.get(url)
logger.info(f"Warmup ping OK: {url}")
except Exception as e:
logger.debug(f"Warmup ping failed for endpoint: {e}")
except Exception as e:
logger.debug(f"Warmup ping skipped: {e}")
_startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
else:
logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
# Keep-alive is opt-in. The ping path performs model discovery, and when
# stale LAN endpoints are configured it can add periodic backend pressure
# that delays unrelated UI requests such as Notes/Documents.
_keepalive_enabled = str(os.getenv("ODYSSEUS_MODEL_KEEPALIVE", "")).lower() in {"1", "true", "yes", "on"}
if _keepalive_enabled:
async def _keepalive_loop():
while True:
_startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
# Warmup: ping all known LLM endpoints to prime connections
async def _warmup_endpoints():
try:
import httpx
# model_discovery has no get_endpoints(); that call raised
# AttributeError every run and silently disabled warmup/keepalive.
# Resolve the /models probe URLs via the real discovery API, off the
# event loop since discovery does a blocking port scan.
urls = (
await asyncio.to_thread(model_discovery.warmup_ping_urls)
if model_discovery else []
)
for url in urls:
try:
await asyncio.sleep(60)
await _warmup_endpoints()
async with httpx.AsyncClient(timeout=5.0) as client:
await client.get(url)
logger.info(f"Warmup ping OK: {url}")
except Exception as e:
logger.warning(f"Keepalive loop error: {e}")
await asyncio.sleep(300) # Back off on error
logger.debug(f"Warmup ping failed for endpoint: {e}")
except Exception as e:
logger.debug(f"Warmup ping skipped: {e}")
_startup_tasks.append(asyncio.create_task(_keepalive_loop()))
_startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
# Keep-alive: ping endpoints every 60 seconds to prevent cold starts
async def _keepalive_loop():
while True:
try:
await asyncio.sleep(60)
await _warmup_endpoints()
except Exception as e:
logger.warning(f"Keepalive loop error: {e}")
await asyncio.sleep(300) # Back off on error
_startup_tasks.append(asyncio.create_task(_keepalive_loop()))
async def _ensure_default_tasks():
# Create/reconcile default automation tasks + personal assistant for every user.
@@ -1270,12 +1171,3 @@ async def _shutdown_event():
except Exception as e:
logger.warning(f"MCP shutdown error: {e}")
logger.info("Application shutdown complete")
if __name__ == "__main__":
import uvicorn
bind_host = os.getenv("APP_BIND", "127.0.0.1")
bind_port = int(os.getenv("APP_PORT", "7000"))
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")
build-windows-portable.ps1
-72
View File
@@ -1,72 +0,0 @@
#Requires -Version 5.1
<#
Build a portable Windows distribution for Odysseus.
Output layout:
dist\Odysseus\Odysseus.exe
dist\Odysseus\static\...
dist\Odysseus\scripts\...
dist\Odysseus\mcp_servers\...
dist\Odysseus\services\hwfit\data\...
The app then keeps using its normal filesystem layout when frozen.
Usage:
powershell -ExecutionPolicy Bypass -File .\build-windows-portable.ps1
#>
$ErrorActionPreference = "Stop"
Set-Location -Path $PSScriptRoot
function Write-Step($msg) { Write-Host ""; Write-Host ("==> " + $msg) -ForegroundColor Cyan }
function Fail($msg) {
Write-Host ""
Write-Host ("ERROR: " + $msg) -ForegroundColor Red
exit 1
}
Write-Step "Checking for Python"
$pyExe = $null
if (Test-Path ".\.venv\Scripts\python.exe") {
$pyExe = (Resolve-Path ".\.venv\Scripts\python.exe").Path
} else {
foreach ($c in @("py", "python")) {
$cmd = Get-Command $c -ErrorAction SilentlyContinue
if ($cmd) { $pyExe = $cmd.Source; break }
}
if ($pyExe -like "*WindowsApps*python.exe") {
$pyCmd = Get-Command py -ErrorAction SilentlyContinue
if ($pyCmd) {
$pyExe = $pyCmd.Source
}
}
}
if (-not $pyExe) {
Fail "Python not found on PATH. Install Python 3.11+ first."
}
Write-Host ("Using Python: " + $pyExe)
Write-Step "Installing build dependencies"
& $pyExe -m pip install --upgrade pip --quiet
& $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow
if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." }
Write-Step "Building portable exe bundle"
Remove-Item -Recurse -Force build, dist -ErrorAction SilentlyContinue
$dataArgs = @(
"--add-data", "static;static",
"--add-data", "scripts;scripts",
"--add-data", "mcp_servers;mcp_servers",
"--add-data", "services/hwfit/data;services/hwfit/data",
"--add-data", "config;config",
"--add-data", ".env.example;.env.example"
)
& $pyExe -m PyInstaller --noconfirm --clean --onedir --noconsole --icon=static/icon.ico --name Odysseus @dataArgs launcher.py
if ($LASTEXITCODE -ne 0) { Fail "PyInstaller build failed." }
Write-Host ""
Write-Host "Build complete." -ForegroundColor Green
Write-Host "Portable app folder: $PSScriptRoot\dist\Odysseus" -ForegroundColor Green
Write-Host "Distribute the whole folder (or zip it) so static assets and scripts stay with the exe." -ForegroundColor Green
companion/routes.py
+3 -17
View File
@@ -5,9 +5,8 @@ offers and pair to it, without duplicating any LLM logic.
Auth is enforced globally by AuthMiddleware (app.py), so reaching a handler here
means the caller is authenticated by either a cookie session or a Bearer `ody_`
API token. Ping/info accept either credential type, models requires a chat-
scoped API token for bearer callers, and the pairing endpoints are admin-cookie
only.
API token. The read endpoints (ping/info/models) accept either; the pairing
endpoints are admin-cookie only.
Pairing CSRF posture: minting happens ONLY on POST. The session cookie is
SameSite=Lax (routes/auth_routes.py), which a browser does not send on a
@@ -19,7 +18,7 @@ on a GET would be unsafe (Lax cookies ride top-level GET navigations), so GET
import html
from fastapi import APIRouter, HTTPException, Request
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from core.middleware import require_admin
@@ -53,18 +52,6 @@ def owner_can_see(row_owner, owner) -> bool:
return row_owner is None or row_owner == owner
def require_models_scope(request: Request) -> None:
"""Require the companion chat scope for bearer-token model inventory."""
if not getattr(request.state, "api_token", False):
return
scopes = getattr(request.state, "api_token_scopes", None) or []
if isinstance(scopes, str):
scopes = [scope.strip() for scope in scopes.split(",")]
scope_set = {str(scope).strip() for scope in scopes if str(scope).strip()}
if _pairing.COMPANION_SCOPE not in scope_set:
raise HTTPException(403, "API token requires chat scope")
def mint_pairing_token(owner: str, invalidate=None) -> tuple[str, str]:
"""Mint a pairing token AND invalidate the auth middleware's in-memory token
cache, so the new token is accepted on the very next request without a server
@@ -116,7 +103,6 @@ def setup_companion_routes() -> APIRouter:
rows -- the same rule as owner_filter. Read-only; never returns api_key
material.
"""
require_models_scope(request)
import json as _json
from core.database import SessionLocal, ModelEndpoint
core/atomic_io.py
-2
View File
@@ -34,8 +34,6 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) ->
def atomic_write_text(path: str, text: str) -> None:
if not isinstance(text, str):
raise TypeError("atomic_write_text expects a string")
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
tmp = f"{path}.tmp.{os.getpid()}"
with open(tmp, "w", encoding="utf-8") as f:
core/auth.py
+18 -34
View File
@@ -20,7 +20,6 @@ logger = logging.getLogger(__name__)
from core.atomic_io import atomic_write_json as _atomic_write_json # noqa: E402
from core.middleware import INTERNAL_TOOL_USER # noqa: E402
DEFAULT_PRIVILEGES = {
"can_use_agent": True,
@@ -48,7 +47,7 @@ ADMIN_PRIVILEGES["allowed_models_restricted"] = False
# backwards for this sentinel.
ADMIN_PRIVILEGES["block_all_models"] = False
from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH
from src.constants import AUTH_FILE
DEFAULT_AUTH_PATH = AUTH_FILE
TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days
@@ -66,7 +65,7 @@ TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days
# of those names would be denied an assistant and inconsistently owner-scoped.
# Refuse to create or rename into any of them so the sentinels can't be
# impersonated. (Keep this in sync with that synthetic-owner set.)
RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"})
RESERVED_USERNAMES = frozenset({"internal-tool", "api", "demo", "system"})
def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]:
@@ -176,17 +175,16 @@ class AuthManager:
)
old_user = "admin"
old_hash = self._config["password_hash"]
with self._config_lock:
self._config = {
"users": {
old_user: {
"password_hash": old_hash,
"created": time.time(),
"is_admin": True,
}
self._config = {
"users": {
old_user: {
"password_hash": old_hash,
"created": time.time(),
"is_admin": True,
}
}
self._save()
}
self._save()
logger.info(f"Migrated single-user auth to multi-user (admin: {old_user})")
def _drop_reserved_loaded_users(self):
@@ -205,9 +203,8 @@ class AuthManager:
continue
normalized[key] = data
if removed or normalized != users:
with self._config_lock:
self._config["users"] = normalized
self._save()
self._config["users"] = normalized
self._save()
if removed:
logger.warning(
"Removed reserved username(s) from auth config: %s",
@@ -246,15 +243,6 @@ class AuthManager:
def is_configured(self) -> bool:
return len(self.users) > 0
def policy(self) -> dict:
"""Return public auth policy constants for the frontend."""
return {
"password_min_length": PASSWORD_MIN_LENGTH,
"reserved_usernames": sorted(RESERVED_USERNAMES),
"signup_enabled": self.signup_enabled,
"session_days": TOKEN_TTL // 86400,
}
# ------------------------------------------------------------------
# Account management
# ------------------------------------------------------------------
@@ -585,20 +573,16 @@ class AuthManager:
return None
return self.create_session_trusted(username)
def create_session_trusted(self, username: str) -> Optional[str]:
def create_session_trusted(self, username: str) -> str:
"""Issue a session token for an already-verified user.
Call only after verify_password (and TOTP if enabled) have passed."""
username = username.strip().lower()
token = secrets.token_hex(32)
with self._config_lock:
if username not in self.users:
logger.warning("Refused to issue session for missing user '%s'", username)
return None
with self._sessions_lock:
self._sessions[token] = {
"username": username,
"expiry": time.time() + TOKEN_TTL,
}
with self._sessions_lock:
self._sessions[token] = {
"username": username,
"expiry": time.time() + TOKEN_TTL,
}
self._save_sessions()
return token
core/database.py
+8 -234
View File
@@ -2,18 +2,12 @@ import os
import logging
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional
from urllib.parse import unquote, urlparse
from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
from sqlalchemy.engine import Engine, make_url
from sqlalchemy.engine import Engine
from sqlalchemy.types import TypeDecorator
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.orm import relationship, sessionmaker, backref
from src.runtime_paths import get_app_root
from core.platform_compat import safe_chmod, IS_WINDOWS
logger = logging.getLogger(__name__)
# Create base class for declarative models
@@ -35,42 +29,9 @@ class TimestampMixin:
def updated_at(cls):
return Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive, nullable=False)
# Ensure the writable data directory exists before SQLite connects.
from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE
Path(DATA_DIR).mkdir(parents=True, exist_ok=True)
def _default_database_url() -> str:
return f"sqlite:///{Path(DATA_DIR) / 'app.db'}"
def _normalize_sqlite_url(url: str) -> str:
"""Resolve relative ordinary SQLite paths without rewriting URI filenames."""
try:
parsed = make_url(url)
except Exception:
return url
if parsed.get_backend_name() != "sqlite":
return url
db_path = parsed.database
if (
not db_path
or db_path == ":memory:"
or str(db_path).lower().startswith("file:")
or os.path.isabs(str(db_path))
):
return url
absolute_path = (Path(get_app_root()) / str(db_path)).resolve().as_posix()
return parsed.set(database=absolute_path).render_as_string(
hide_password=False
)
# Get database URL from environment, default to SQLite in DATA_DIR
DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database_url()))
from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE
DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DATA_DIR}/app.db")
# Create engine
engine = create_engine(
@@ -78,59 +39,6 @@ engine = create_engine(
connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
)
# Sidecar files SQLite can create next to the main DB. -journal is the default
# rollback journal; -wal/-shm appear once WAL is enabled. Each can hold copies of
# secret-bearing pages, so they get the same 0o600 lockdown as the DB itself.
_SQLITE_SIDECARS = ("-journal", "-wal", "-shm")
def _sqlite_db_path(url) -> Optional[str]:
"""Return the filesystem path for a file-backed SQLite URL.
SQLite query parameters such as ``mode=memory`` only affect filename
semantics when SQLAlchemy enables URI handling with ``uri=true``. Ordinary
file URLs must therefore remain file-backed even when they contain a query
parameter named ``mode``.
For SQLite ``file:`` URIs, an empty authority or ``localhost`` identifies a
local path. Other authorities are retained as UNC-style paths.
"""
if url.get_backend_name() != "sqlite":
return None
db_path = url.database
if not db_path or db_path == ":memory:":
return None
db_path = str(db_path)
query = {
str(key).lower(): str(value).strip().lower()
for key, value in dict(getattr(url, "query", {}) or {}).items()
}
uri_enabled = query.get("uri") in {"1", "true", "yes", "on"}
is_file_uri = db_path.lower().startswith("file:")
if not uri_enabled or not is_file_uri:
return db_path
if (
db_path.lower().startswith("file::memory:")
or query.get("mode") == "memory"
):
return None
parsed = urlparse(db_path)
fs_path = parsed.path or ""
if not fs_path or fs_path == ":memory:":
return None
authority = parsed.netloc
if authority and authority.lower() != "localhost":
fs_path = f"//{authority}{fs_path}"
return unquote(fs_path)
# Create session factory
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
@@ -348,7 +256,6 @@ class GalleryImage(TimestampMixin, Base):
id = Column(String, primary_key=True, index=True)
filename = Column(String, nullable=False, unique=True)
prompt = Column(Text, nullable=False, default="")
caption = Column(Text, nullable=True, default="")
model = Column(String, nullable=True)
size = Column(String, nullable=True)
quality = Column(String, nullable=True)
@@ -1255,29 +1162,6 @@ def _migrate_add_multiuser_owner_columns():
_migrate_add_owner_to_table("documents", "ix_documents_owner")
def _migrate_add_gallery_caption_column():
"""Add OCR/vision caption storage for gallery images."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
columns = [row[1] for row in conn.execute("PRAGMA table_info(gallery_images)").fetchall()]
if columns and "caption" not in columns:
conn.execute("ALTER TABLE gallery_images ADD COLUMN caption TEXT DEFAULT ''")
conn.commit()
logging.getLogger(__name__).info("Migrated: added caption column to gallery_images")
except Exception as e:
logging.getLogger(__name__).warning(f"Migration gallery caption column failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_api_token_scopes_column():
"""Add API token scopes for existing installs.
@@ -1766,7 +1650,6 @@ class CalendarEvent(TimestampMixin, Base):
# `Z`-suffix on serialization so the frontend interprets correctly.
is_utc = Column(Boolean, default=False, nullable=False)
rrule = Column(String, default="")
recurrence_exdates = Column(Text, default="") # JSON list of skipped occurrence starts
color = Column(String, nullable=True) # per-event color override
status = Column(String, default="confirmed") # confirmed, cancelled
importance = Column(String, default="normal") # low | normal | high | critical
@@ -1891,41 +1774,6 @@ def init_db():
"""
_migrate_model_endpoints()
Base.metadata.create_all(bind=engine)
# Lock the DB file (and any SQLite sidecars) to 0o600 — it holds bearer-token
# + bcrypt hashes and encrypted provider keys. POSIX only; safe_chmod no-ops
# on Windows (ACL-restricted profile dir) and the path helper returns None for
# Postgres / in-memory. Must stay AFTER create_all: the file is born here at
# the umask default, and nothing below resets the mode. The path comes from
# engine.url (SQLAlchemy's parsed URL), so a driver-qualified or query-tagged
# DATABASE_URL still resolves to the real file instead of slipping through.
db_path = _sqlite_db_path(engine.url)
if db_path is not None:
# Fail closed-loud on the main file: this is the only access control on
# it, so if the chmod genuinely fails (read-only FS, foreign owner) an
# operator should hear about it. safe_chmod also returns False as a
# Windows no-op, so guard on IS_WINDOWS to avoid a spurious warning there.
if not safe_chmod(db_path, 0o600) and not IS_WINDOWS:
logger.warning(
"Could not restrict %s to 0o600; it holds secrets and may be "
"world-readable. Check filesystem permissions and ownership.",
db_path,
)
# Re-lock any sidecars present at startup. New ones inherit the main
# file's mode (now 0o600, since we set it first), and they're usually
# absent here, but a stale -wal/-shm/-journal left by an older 0o644
# install could still expose secret pages. Absent sidecars are the
# normal case, not an error — only a failed chmod warrants a warning.
for suffix in _SQLITE_SIDECARS:
sidecar = db_path + suffix
if (
os.path.exists(sidecar)
and not safe_chmod(sidecar, 0o600)
and not IS_WINDOWS
):
logger.warning(
"Could not restrict %s to 0o600; it may expose DB pages.",
sidecar,
)
_migrate_add_hidden_models_column()
_migrate_add_cached_models_column()
_migrate_add_pinned_models_column()
@@ -1943,7 +1791,6 @@ def init_db():
_migrate_add_token_columns()
_migrate_add_mode_column()
_migrate_add_multiuser_owner_columns()
_migrate_add_gallery_caption_column()
_migrate_add_api_token_scopes_column()
_migrate_backfill_document_owner_from_session()
_migrate_assign_legacy_owner()
@@ -1966,7 +1813,6 @@ def init_db():
_migrate_add_calendar_origin()
_migrate_add_calendar_account_id()
_migrate_add_caldav_sync_columns()
_migrate_add_calendar_recurrence_exdates()
_migrate_chat_messages_fts()
_migrate_encrypt_email_passwords()
_migrate_encrypt_signatures()
@@ -2011,20 +1857,6 @@ def _migrate_chat_messages_fts():
conn = None
try:
conn = sqlite3.connect(db_path)
fts_content_expr_new = (
"CASE WHEN instr(COALESCE(new.content, ''), ';base64,') > 0 "
"OR instr(COALESCE(new.content, ''), 'data:image/') > 0 "
"OR instr(COALESCE(new.content, ''), 'data:audio/') > 0 "
"THEN '[inline media omitted from search index]' "
"ELSE COALESCE(new.content, '') END"
)
fts_content_expr_cm = (
"CASE WHEN instr(COALESCE(cm.content, ''), ';base64,') > 0 "
"OR instr(COALESCE(cm.content, ''), 'data:image/') > 0 "
"OR instr(COALESCE(cm.content, ''), 'data:audio/') > 0 "
"THEN '[inline media omitted from search index]' "
"ELSE COALESCE(cm.content, '') END"
)
try:
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS temp._odysseus_fts5_probe USING fts5(content)")
conn.execute("DROP TABLE IF EXISTS temp._odysseus_fts5_probe")
@@ -2033,7 +1865,7 @@ def _migrate_chat_messages_fts():
return
conn.executescript(
f"""
"""
CREATE VIRTUAL TABLE IF NOT EXISTS chat_messages_fts USING fts5(
content,
message_id UNINDEXED,
@@ -2041,14 +1873,10 @@ def _migrate_chat_messages_fts():
role UNINDEXED
);
DROP TRIGGER IF EXISTS chat_messages_fts_ai;
DROP TRIGGER IF EXISTS chat_messages_fts_ad;
DROP TRIGGER IF EXISTS chat_messages_fts_au;
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ai
AFTER INSERT ON chat_messages BEGIN
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role);
VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
END;
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ad
@@ -2060,14 +1888,14 @@ def _migrate_chat_messages_fts():
AFTER UPDATE ON chat_messages BEGIN
DELETE FROM chat_messages_fts WHERE message_id = old.id;
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role);
VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
END;
"""
)
conn.execute(
f"""
"""
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
SELECT COALESCE(cm.content, ''), cm.id, cm.session_id, cm.role
FROM chat_messages cm
WHERE NOT EXISTS (
SELECT 1 FROM chat_messages_fts fts
@@ -2075,7 +1903,6 @@ def _migrate_chat_messages_fts():
)
"""
)
_scrub_legacy_chat_message_fts_media(conn)
conn.commit()
except Exception as e:
logging.getLogger(__name__).warning(f"chat_messages FTS migration failed: {e}")
@@ -2086,37 +1913,6 @@ def _migrate_chat_messages_fts():
pass
def _scrub_legacy_chat_message_fts_media(conn) -> None:
"""Replace already-indexed inline media rows with searchable text only."""
try:
from src.attachment_refs import search_index_text
except Exception as e:
logging.getLogger(__name__).warning(f"chat_messages FTS media scrub skipped: {e}")
return
try:
rows = conn.execute(
"""
SELECT id, session_id, role, content
FROM chat_messages
WHERE instr(COALESCE(content, ''), ';base64,') > 0
OR instr(COALESCE(content, ''), 'data:image/') > 0
OR instr(COALESCE(content, ''), 'data:audio/') > 0
"""
).fetchall()
for message_id, session_id, role, content in rows:
conn.execute("DELETE FROM chat_messages_fts WHERE message_id = ?", (message_id,))
conn.execute(
"""
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
VALUES (?, ?, ?, ?)
""",
(search_index_text(content), message_id, session_id, role),
)
except Exception as e:
logging.getLogger(__name__).warning(f"chat_messages FTS media scrub failed: {e}")
def _migrate_add_email_smtp_security():
"""Add explicit SMTP security mode for Proton Bridge/custom local SMTP."""
import sqlite3
@@ -2368,28 +2164,6 @@ def _migrate_add_calendar_metadata():
except Exception:
pass
def _migrate_add_calendar_recurrence_exdates():
"""Add skipped recurrence occurrences for deleting one instance of a series."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
if columns and "recurrence_exdates" not in columns:
conn.execute("ALTER TABLE calendar_events ADD COLUMN recurrence_exdates TEXT DEFAULT ''")
conn.commit()
except Exception as e:
logging.getLogger(__name__).warning(f"calendar_events recurrence_exdates migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def get_db():
"""
Dependency to get a database session.
core/exceptions.py
+1 -1
View File
@@ -1,4 +1,4 @@
# core/exceptions.py
# src/exceptions.py
"""Custom exceptions for the application."""
class SessionNotFoundError(Exception):
core/log_safety.py
-27
View File
@@ -1,27 +0,0 @@
"""Helpers for keeping sensitive data out of logs.
Endpoint URLs configured by admins can embed credentials in the userinfo
(``https://user:pass@host``) or query string (``?api_key=...``). Logging them
raw leaks those secrets, so route/diagnostic logs run URLs through
``redact_url`` first. Reconstructing the URL without userinfo/query/fragment
also doubles as a sanitizer barrier for CodeQL's clear-text-logging query.
"""
from urllib.parse import urlparse, urlunparse
def redact_url(url: str) -> str:
"""Return a URL safe for logs by removing userinfo and query/fragment.
Keeps scheme, host, port and path so logs stay useful for debugging.
"""
try:
parsed = urlparse(url or "")
host = parsed.hostname or ""
if ":" in host: # IPv6 literal — re-bracket so host:port stays unambiguous
host = f"[{host}]"
if parsed.port:
host = f"{host}:{parsed.port}"
return urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
except Exception:
return "<endpoint>"
core/middleware.py
+8 -7
View File
@@ -15,8 +15,6 @@ from starlette.responses import Response
# same value from this module. Never persisted or exposed externally.
INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32)
INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token"
# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved.
INTERNAL_TOOL_USER = "internal-tool"
def is_cors_preflight(method: str, headers) -> bool:
@@ -41,7 +39,7 @@ def require_admin(request: Request):
hdr = request.headers.get(INTERNAL_TOOL_HEADER)
if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN):
return
if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER:
if getattr(request.state, "current_user", None) == "internal-tool":
return
except Exception:
pass
@@ -67,9 +65,10 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
response = await call_next(request)
path = request.url.path
# Tool render endpoints
# Tool render endpoints are served inside iframes — allow framing by self
is_tool_render = path.startswith("/api/tools/") and path.endswith("/render")
# Document library PDF preview endpoint
# PDF previews are embedded by the in-app document library. Keep the
# exception route-scoped so normal app pages remain unframeable.
is_document_pdf_preview = path.startswith("/api/document/") and path.endswith("/render-pdf")
# Visual report pages are self-contained HTML — need inline scripts + external images
is_report = path.startswith("/api/research/report/")
@@ -96,7 +95,9 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
"frame-ancestors 'none'"
)
elif is_tool_render:
# Skip framing headers for tools.
# Tool iframe content: skip all framing headers — the iframe's
# sandbox="allow-scripts" attribute provides isolation.
# Don't overwrite the route's own restrictive CSP either.
pass
elif is_document_pdf_preview:
response.headers["X-Frame-Options"] = "SAMEORIGIN"
@@ -117,7 +118,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
f"script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net; "
"style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
"font-src 'self' https://cdn.jsdelivr.net; "
"img-src 'self' data: blob: https:; "
"img-src 'self' data: blob:; "
"media-src 'self' blob:; "
"connect-src 'self'; "
"frame-src 'self'; "
core/session_manager.py
+22 -59
View File
@@ -16,8 +16,6 @@ from typing import Dict, Optional
from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive
from .models import Session, ChatMessage
from src.attachment_refs import persistable_message_content
from src.upload_handler import reserve_message_upload_references
# Re-export singleton accessors from models for convenience
from .models import set_session_manager_instance, get_session_manager_instance
@@ -42,18 +40,7 @@ def _parse_msg_content(raw):
if isinstance(raw, str) and raw.startswith('[{') and '"type"' in raw:
try:
parsed = json.loads(raw)
# Only treat as serialized multimodal content when EVERY element is
# a dict whose "type" is a recognized content-block kind. Otherwise a
# plain text message that merely *looks* like a JSON array of objects
# (e.g. a user pasting an API schema/sample with a "type" field) was
# silently parsed back into a list, destroying the original string.
_BLOCK_TYPES = {
"text", "image", "image_url", "audio", "input_audio",
"input_image", "document", "file",
}
if (isinstance(parsed, list) and parsed
and all(isinstance(p, dict) and p.get("type") in _BLOCK_TYPES
for p in parsed)):
if isinstance(parsed, list) and all(isinstance(p, dict) for p in parsed):
return parsed
except (json.JSONDecodeError, ValueError):
pass
@@ -74,7 +61,6 @@ class SessionManager:
def __init__(self, sessions_file: str = None):
# sessions_file kept for backward compat, not used
self.sessions: Dict[str, Session] = {}
self.upload_handler = None
self.load_sessions()
# ------------------------------------------------------------------
@@ -233,26 +219,17 @@ class SessionManager:
logger.warning("Dropping message for deleted session %s", session_id)
return
missing_upload_id = reserve_message_upload_references(
getattr(self, "upload_handler", None),
getattr(db_session, "owner", None),
message.content,
message.metadata,
)
if missing_upload_id:
raise ValueError(
f"Referenced upload is no longer available: {missing_upload_id}"
)
msg_id = str(uuid.uuid4())
msg_time = datetime.utcnow()
if message.metadata is None:
message.metadata = {}
message.metadata.setdefault('timestamp', _message_timestamp_iso(msg_time))
# Multimodal content may contain provider data URLs for the live
# model call. Persist only readable text plus attachment references
# so chat_messages/FTS do not duplicate upload bytes.
_content = persistable_message_content(message.content, message.metadata)
# Multimodal content (image/audio attachments) is a list — serialize
# to JSON so the Text column can store it. On reload, _db_to_session
# detects the JSON-array prefix and parses it back.
_content = message.content
if isinstance(_content, list):
_content = json.dumps(_content)
db_message = DbChatMessage(
id=msg_id,
session_id=session_id,
@@ -334,28 +311,6 @@ class SessionManager:
session = self.get_session(session_id)
db = SessionLocal()
try:
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session is None:
logger.warning("Cannot replace history for missing session %s", session_id)
return False
# Reserve every incoming attachment before removing any durable
# message row. reserve_upload() shares the upload lifecycle lock
# with cleanup, so an upload cannot be deleted between this
# ownership check/access touch and the replacement transaction.
# A failed reservation must leave the existing transcript intact.
for message in messages:
missing_upload_id = reserve_message_upload_references(
getattr(self, "upload_handler", None),
getattr(db_session, "owner", None),
message.content,
message.metadata,
)
if missing_upload_id:
raise ValueError(
f"Referenced upload is no longer available: {missing_upload_id}"
)
db.query(DbChatMessage).filter(DbChatMessage.session_id == session_id).delete()
now = datetime.now(timezone.utc)
for i, message in enumerate(messages):
@@ -364,9 +319,15 @@ class SessionManager:
id=msg_id,
session_id=session_id,
role=message.role,
# Mirrors _persist_message: keep raw media bytes out of the
# persisted transcript and search index.
content=persistable_message_content(message.content, message.metadata),
# Multimodal content (image/audio attachments) is a list;
# serialize to JSON so the Text column round-trips via
# _parse_msg_content. Storing the raw list let SQLAlchemy
# bind its single-quoted repr, which _parse_msg_content
# cannot parse (it looks for double-quoted "type"), so the
# attachment was destroyed on reload. Mirrors _persist_message.
content=(json.dumps(message.content)
if isinstance(message.content, list)
else message.content),
meta_data=json.dumps(message.metadata) if message.metadata else None,
timestamp=now + timedelta(microseconds=i),
)
@@ -375,10 +336,12 @@ class SessionManager:
message.metadata = {}
message.metadata["_db_id"] = msg_id
db_session.message_count = len(messages)
db_session.updated_at = now
db_session.last_accessed = now
db_session.last_message_at = now
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(messages)
db_session.updated_at = now
db_session.last_accessed = now
db_session.last_message_at = now
db.commit()
session.history = list(messages)
docker-compose.gpu-amd.yml
-7
View File
@@ -60,13 +60,6 @@ services:
- ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
- ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
- ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
- ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
- ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
- ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
- ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
- DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
- GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
docker-compose.gpu-nvidia.yml
-7
View File
@@ -59,13 +59,6 @@ services:
- ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
- ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
- ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
- ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
- ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
- ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
- ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
- DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
- GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
docker-compose.yml
-7
View File
@@ -48,13 +48,6 @@ services:
- ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
- ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
- ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
- ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
- ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
- ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
- ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
- DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
- GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
docker/build-realesrgan-wheels.sh
-70
View File
@@ -1,70 +0,0 @@
#!/usr/bin/env bash
# Build patched wheels for Real-ESRGAN's unmaintained dependencies.
#
# basicsr / gfpgan / facexlib (xinntao, last released 2022) read their version
# in setup.py with:
#
# exec(compile(f.read(), version_file, 'exec'))
# return locals()['__version__']
#
# Python 3.13+ implements PEP 667: locals() inside a function returns an
# independent snapshot that exec() can no longer mutate, so the read raises
# `KeyError: '__version__'` and the sdist build fails. That is why the Cookbook
# "install realesrgan" button dies on the python:3.14 image. The packages have
# no fixed release, so we patch get_version() to exec into an explicit namespace
# dict (works on every Python) and build wheels from the patched source.
#
# Usage: build-realesrgan-wheels.sh [OUTPUT_DIR] (default: /wheels)
set -euo pipefail
OUT="${1:-/wheels}"
mkdir -p "$OUT"
work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT
cd "$work"
# Pinned to the versions Real-ESRGAN 0.3.0 resolves to.
SPECS="basicsr==1.4.2 gfpgan==1.3.8 facexlib==0.3.0"
for spec in $SPECS; do
name="${spec%%==*}"
ver="${spec##*==}"
# pip download builds metadata (and trips the same bug), so fetch the raw
# sdist URL from the PyPI JSON API instead.
url="$(python - "$name" "$ver" <<'PY'
import json, sys, urllib.request
name, ver = sys.argv[1], sys.argv[2]
data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{ver}/json"))
for f in data["urls"]:
if f["packagetype"] == "sdist":
print(f["url"]); break
else:
sys.exit(f"no sdist found for {name}=={ver}")
PY
)"
echo ">> fetching ${name} ${ver}: ${url}"
curl -fsSL "$url" -o "${name}.tar.gz"
tar xzf "${name}.tar.gz"
done
echo ">> patching get_version()"
python - <<'PY'
import pathlib
old_exec = "exec(compile(f.read(), version_file, 'exec'))"
new_exec = "_ver_ns = {}\n exec(compile(f.read(), version_file, 'exec'), _ver_ns)"
old_ret = "return locals()['__version__']"
new_ret = "return _ver_ns['__version__']"
patched = 0
for setup in pathlib.Path(".").glob("*/setup.py"):
s = setup.read_text()
if old_exec in s and old_ret in s:
setup.write_text(s.replace(old_exec, new_exec).replace(old_ret, new_ret))
print(" patched", setup)
patched += 1
assert patched == 3, f"expected to patch 3 setup.py files, patched {patched}"
PY
echo ">> building wheels into ${OUT}"
pip wheel --no-deps -w "$OUT" ./basicsr-* ./gfpgan-* ./facexlib-*
ls -l "$OUT"
docker/entrypoint.sh
+19 -74
View File
@@ -13,8 +13,6 @@ set -e
PUID="${PUID:-1000}"
PGID="${PGID:-1000}"
GOSU_BIN="$(command -v gosu)"
PYTHON_BIN="$(command -v python)"
# Reuse an existing matching group/user if the host's UID/GID already
# corresponds to one in /etc/passwd (e.g. when the image is rebuilt
@@ -26,78 +24,26 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then
useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus
fi
ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)"
[ -z "$ODY_USER" ] && ODY_USER=odysseus
# Docker-socket group plumbing for the explicit host-Docker overlay. When
# opted in, the socket is owned by root:<host docker gid>. Add the app user
# to that group and later call gosu by username so supplementary groups are
# retained.
DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}"
if [ "${ODYSSEUS_ENABLE_HOST_DOCKER:-}" = "true" ] && [ -S "$DOCKER_SOCK" ]; then
SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')"
if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then
if ! getent group "$SOCK_GID" >/dev/null 2>&1; then
groupadd -g "$SOCK_GID" docker_host || true
fi
SOCK_GROUP="$(getent group "$SOCK_GID" | cut -d: -f1)"
if [ -n "$SOCK_GROUP" ]; then
usermod -aG "$SOCK_GROUP" "$ODY_USER" 2>/dev/null || true
fi
fi
fi
mount_root_for() {
awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true
}
is_broad_mount_root() {
case "$1" in
/|/home|/srv|/var|/usr|/opt|/tmp|/mnt|/media)
return 0
;;
esac
return 1
}
repair_tree_ownership() {
dir="$1"
# Repair ownership on every writable path the app touches at runtime.
#
# Bind-mounted dirs (/app/data, /app/logs) are the obvious ones, but
# the app ALSO writes inside the image's own source tree at runtime:
# - services/cache/{search,content}/* (search cache LRU)
# - services/search_analytics.json
# - services/search_engine_error.log
# - services/tts cache, etc.
# These dirs were created as root during `docker build`, so dropping
# to PUID:PGID would otherwise crash on the first import that tries
# to mkdir them. Chown the whole /app tree — fast (<1s on this size)
# and idempotent via the `-not -uid` filter so we only touch files
# that need fixing.
for dir in /app /app/data /app/logs; do
if [ -d "$dir" ]; then
find "$dir" -xdev -not -uid "$PUID" -print0 2>/dev/null \
# `find ... -not -uid` keeps this O(touched-files), not
# O(everything), so terabyte-sized maildirs don't slow startup.
find "$dir" -not -uid "$PUID" -print0 2>/dev/null \
| xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true
fi
}
repair_app_tree_ownership() {
if [ -d /app ]; then
find /app -xdev \
\( -path /app/data -o -path /app/logs -o -path /app/.ssh -o -path /app/.cache -o -path /app/.local \) -prune \
-o -not -uid "$PUID" -print0 2>/dev/null \
| xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true
fi
}
repair_bind_mount_ownership() {
dir="$1"
if [ ! -d "$dir" ]; then
return
fi
mount_root="$(mount_root_for "$dir")"
if is_broad_mount_root "$mount_root"; then
echo "Skipping recursive ownership repair for $dir because it maps to broad host path $mount_root" >&2
chown "$PUID:$PGID" "$dir" 2>/dev/null || true
return
fi
repair_tree_ownership "$dir"
}
# Repair image-owned writable paths without walking into bind-mounted host
# trees, then repair the app-owned mount roots separately.
repair_app_tree_ownership
for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do
repair_bind_mount_ownership "$dir"
done
# Cookbook installs vllm/etc. via `pip install --user`, which pulls
@@ -124,7 +70,6 @@ for cu in \
break
fi
done
# Disable the FlashInfer JIT sampler unconditionally — it is sampler-only
# and has no impact on the attention path, but requires nvcc + matching
# CUDA headers at startup. Without this, vLLM crashes with "Could not find
@@ -138,9 +83,9 @@ export PATH="/app/.local/bin:$PATH"
# Run first-time setup as the app user so data/ files get the right ownership.
# setup.py is idempotent — skips auth.json / .env if they already exist.
# || true so a setup failure never prevents the container from starting.
"$GOSU_BIN" "$ODY_USER" "$PYTHON_BIN" /app/setup.py || true
gosu "$PUID:$PGID" python /app/setup.py || true
# Drop root and run the actual app. `gosu` is preferred over `su` /
# `sudo` because it cleans up the process tree (no extra shell layer)
# so signals (SIGTERM from `docker stop`) reach uvicorn directly.
exec "$GOSU_BIN" "$ODY_USER" "$@"
exec gosu "$PUID:$PGID" "$@"
docker/host-docker.yml
-12
View File
@@ -1,12 +0,0 @@
# High-trust host Docker access. Enable only when local Docker-daemon
# management from Cookbook is required and you accept that raw socket access
# grants broad control over the host Docker daemon.
# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
# DOCKER_GID=<numeric host Docker group id>
services:
odysseus:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
group_add: ["${DOCKER_GID:-963}"]
environment:
- ODYSSEUS_ENABLE_HOST_DOCKER=true
docs/attachments.md
-85
View File
@@ -1,85 +0,0 @@
# Attachment References and Upload Storage
Odysseus stores uploaded bytes once under the configured upload directory and
passes stable references through chat history, tools, and future artifact work.
The goal is to avoid duplicating large inline media payloads in
`chat_messages.content` or the SQLite FTS index.
## Reference Shape
Attachment references use this minimum shape:
```json
{
"type": "attachment_ref",
"attachment_id": "32hex-or-32hex.ext",
"name": "original-filename.png",
"mime": "image/png",
"size": 12345,
"checksum_sha256": "hex-digest",
"created_at": "2026-07-09T12:00:00"
}
```
Optional fields such as `width`, `height`, `vision`, `vision_model`, and
`gallery_id` may be present when the uploader or preprocessing path knows them.
## Persistence
The live model call may still receive provider-specific multimodal blocks for
the current turn. Persistence is different:
- `chat_messages.content` stores readable text plus compact attachment reference
lines, never raw `data:*;base64,...` upload bytes.
- `chat_messages.metadata.attachments` stores structured attachment reference
metadata for UI reloads and future processing.
- The SQLite FTS migration recreates chat-message FTS triggers so new rows do
not index inline media payloads, and it scrubs legacy rows that were already
indexed with data URLs.
## Tool Access
Agent/tool context receives upload entries as `attachment_ref` manifests with an
`odysseus://attachment/<id>` URI and `read_policy: "owner_checked_upload"`.
For compatibility with existing built-in tools, a local `path` may be included
only after all of these checks pass:
- the upload ID resolves through `UploadHandler.resolve_upload`;
- the requested owner is allowed to read the upload;
- the file remains inside the configured upload directory;
- the file path is inside the tool-readable roots.
External MCP/custom tools should treat the URI and attachment ID as the stable
contract and request bytes through an owner-checked server path, not by assuming
host filesystem layout.
## Retention and Deletion
Current retention behavior is conservative:
- uploads are indexed in `uploads.json` with owner, checksum, MIME type, size,
and creation time;
- admin cleanup first scans persisted chat metadata/content, document versions,
PDF source markers, gallery hashes, notes, and calendar records for live
references;
- cleanup fails closed if that reference scan cannot complete, and the lower-level
cleanup API removes nothing unless it receives a complete reference snapshot;
- expired, unreferenced uploads are removed during the completed scan, while
attachment-bearing writers must first take an owner-checked reservation that
serializes with deletion and refreshes the upload's access timestamp;
- deliberate removal atomically drops matching `uploads.json` rows before deleting
the bytes and restores those rows if filesystem removal fails;
- deleting a chat removes the chat rows but does not immediately delete shared
upload bytes, because the same upload may also be referenced by gallery items,
documents, duplicate-upload rows, or future artifact records.
There is no distinct artifact table in the current schema. Artifact-like upload
references persisted in chat or document text are covered by the canonical
attachment-ID scan; any future artifact store must be added to reference discovery
before cleanup is allowed to consider its uploads unreferenced.
Cleanup and write reservations share the upload-index lock. This closes the
scan/write/delete race in the documented single-worker deployment; a future
multi-process deployment must add an inter-process lock or move lifecycle state
into the database before enabling destructive cleanup in more than one worker.
docs/odysseus-browser.jpg
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 185 KiB

docs/odysseus.jpg
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 79 KiB

After

Width:  |  Height:  |  Size: 52 KiB

docs/setup.md
+2 -97
View File
@@ -15,7 +15,7 @@ On first setup, Odysseus creates an admin account (`admin` unless
For Docker installs, the same line is in `docker compose logs odysseus`.
Use that for the first login, then change it in **Settings**.
Contributing? See [CONTRIBUTING.md](../CONTRIBUTING.md) for setup, testing, and
Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and
pull request guidelines.
### Docker (recommended)
@@ -99,33 +99,6 @@ Odysseus SSH key and add the public key to the remote server's
ssh-copy-id -i data/ssh/id_ed25519.pub user@server
```
**Host Docker access (explicit opt-in).** Default Docker Compose intentionally
does not mount `/var/run/docker.sock`. You can still connect Odysseus to
existing Ollama, vLLM, and other OpenAI-compatible endpoints without Docker
socket access.
Cookbook/local Docker-daemon management requires the opt-in overlay below. Raw
Docker socket access is high-trust because it can effectively grant broad
control over the host Docker daemon. Remote server Docker workflows over SSH
remain preferred.
Place these values in `.env`, or export them in the shell before running
`docker compose`:
```bash
COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
DOCKER_GID=<host docker group gid>
```
Combine host Docker access with a GPU overlay when both are intentionally
required:
```bash
COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
# or
COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
```
**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
only detect GPUs that Docker exposes to the container — if the host runtime or
device passthrough is not configured, Cookbook sees the iGPU, another card, or
@@ -150,51 +123,6 @@ scripts/check-docker-gpu.sh --enable-nvidia-overlay
# Full assisted setup — install toolkit, then enable overlay if passthrough works:
scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay
```
#### Arch Linux NVIDIA Docker notes
On Arch Linux, verify the host NVIDIA driver and Docker GPU passthrough before enabling the Odysseus NVIDIA overlay.
Install the required packages:
```bash
sudo pacman -Syu
sudo pacman -S docker docker-compose nvidia-container-toolkit nvidia-utils
sudo systemctl enable --now docker
```
Configure Docker to use the NVIDIA container runtime:
```bash
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker
```
Verify the host GPU:
```bash
nvidia-smi
```
Verify Docker GPU passthrough:
```bash
docker run --rm --gpus all nvidia/cuda:12.9.0-base-ubuntu22.04 nvidia-smi
```
Then enable the Odysseus NVIDIA compose overlay:
```env
COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml
```
Rebuild and verify the GPU inside the Odysseus container:
```bash
docker compose up -d --build
docker compose exec odysseus nvidia-smi -L
```
For first-time local model testing on 8 GB laptop GPUs, start with GGUF/Q4 models on llama.cpp before trying GPTQ/AWQ models on vLLM or SGLang. This keeps the first run simpler while confirming GPU passthrough works.
Safety notes:
- The app never installs host GPU runtime automatically.
@@ -322,19 +250,6 @@ python -m uvicorn app:app --host 127.0.0.1 --port 7000
If `python` points at an older interpreter, use `py -3.12` (or another installed
3.11+ version) for the venv step.
**Exposing on a LAN/Tailscale (Windows):** the launcher binds to `127.0.0.1` and
does **not** read `APP_BIND` / `ODYSSEUS_HOST` from `.env`, so editing `.env`
alone leaves the native Windows server on loopback. Pass the launcher's
`-BindHost` flag instead:
```powershell
powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 -BindHost 0.0.0.0
```
The manual `uvicorn` command takes the same address as `--host 0.0.0.0`. Bind
outside loopback only for a trusted LAN/VPN such as Tailscale: keep
`AUTH_ENABLED=true` and do not expose the port directly to the public internet.
**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents,
email, calendar, deep research) runs fully native. For full **Cookbook** background
model downloads and the agent shell tool, also install
@@ -371,16 +286,6 @@ To expose Odysseus on a local network or Tailscale with HTTPS:
```
4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings).
### Common self-host traps (30-second fixes)
A grab-bag of small gotchas that otherwise turn into long debugging sessions.
- **`AUTH_ENABLED=false` is ignored / you're still forced to log in (Windows).** If you edited `.env` in Notepad it may have saved a UTF-8 **BOM**, turning the first key into `AUTH_ENABLED` so it is never matched. Odysseus loads `.env` with `encoding="utf-8-sig"` to tolerate a leading BOM, but the safe fix is to re-save `.env` as **UTF-8 without BOM** (VS Code: *Save with Encoding → UTF-8*).
- **macOS: the app isn't at `http://localhost:7000`.** macOS AirPlay Receiver usually holds port `7000`, so the macOS start script serves on **`7860`** instead — open `http://localhost:7860`. To use `7000`, free it (System Settings → General → AirDrop & Handoff → turn off *AirPlay Receiver*) and set `APP_PORT=7000`.
- **Copy buttons do nothing over a plain-HTTP Tailscale/LAN URL.** Browsers only expose the clipboard API (`navigator.clipboard`) on **secure origins** — HTTPS, or `localhost`. Over `http://100.x.y.z:7860` it is blocked. Serve over HTTPS (see *HTTPS + LAN/Tailscale exposure* above); `localhost` is exempt, so copy still works on the host itself.
- **Self-hosted ntfy reminders don't reach your phone.** Two things: (1) the bundled ntfy binds to loopback by default — to reach it from your phone set `NTFY_BIND` to your host/Tailscale IP and `NTFY_BASE_URL` to the same server URL in `.env`, then recreate the ntfy container (see the `NTFY_*` block in `.env.example`); (2) in the ntfy **Android** app, subscribe to the topic with **Instant delivery** enabled — non-`ntfy.sh` servers don't get instant push otherwise.
- **Local mail (Dovecot) login fails: "Plaintext authentication disallowed on non-encrypted connections."** Your IMAP/SMTP server is refusing cleartext auth over an unencrypted link. Prefer enabling TLS on the mail server; on a trusted LAN only, you can allow cleartext (Dovecot: `disable_plaintext_auth = no`).
- **Calendar/contacts (Radicale) won't sync.** Point Odysseus at the **full collection URL** with its trailing slash — e.g. `http://host:5232/<user>/<collection-id>/` — not just the server root. Radicale shows this address for each calendar/address book in its web UI.
### Optional Dependencies
`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default.
@@ -517,4 +422,4 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum
`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
To back up or restore everything in `data/`, see the
[Backup & Restore guide](backup-restore.md).
[Backup & Restore guide](docs/backup-restore.md).
launch-windows.ps1
-8
View File
@@ -105,14 +105,6 @@ if (-not $pyExe) {
}
}
if ($pyExe -like "*WindowsApps*python.exe") {
$pyCmd = Get-Command py -ErrorAction SilentlyContinue
if ($pyCmd) {
$pyExe = $pyCmd.Source
$pyArgs = @("-3.11")
}
}
if (-not $pyExe) {
Fail "Couldn't find Python 3.11+ for Windows setup. Install Python 3.11+ (or open the Python launcher with 'py -3.11') from https://www.python.org/downloads/, then re-run this script."
}
launcher.py
-142
View File
@@ -1,142 +0,0 @@
# launcher.py
"""Dedicated entrypoint for the standalone Windows portable launcher.
Handles:
- Immediate GUI splash screen creation using tkinter.
- Suppressing console stream crashes in windowed GUI mode via NullWriter.
- Spawning system tray icon via pystray and Pillow (lazy-loaded).
- Auto-opening default browser pointing to the running backend.
- Launching the FastAPI server (importing and running app.py).
"""
import os
import sys
import threading
import time
import webbrowser
# Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode
class NullWriter:
def write(self, text):
pass
def flush(self):
pass
def isatty(self):
return False
if sys.stdout is None:
sys.stdout = NullWriter()
if sys.stderr is None:
sys.stderr = NullWriter()
splash_root = None
# If running from a frozen PyInstaller bundle, launch the splash screen IMMEDIATELY
if getattr(sys, 'frozen', False):
import tkinter as tk
def show_splash_instantly():
global splash_root
try:
splash_root = tk.Tk()
splash_root.title("Odysseus")
splash_root.overrideredirect(True)
splash_root.configure(bg="#1a1c23")
# Accented borders
splash_root.config(highlightbackground="#e06c75", highlightcolor="#e06c75", highlightthickness=1)
w, h = 360, 160
ws = splash_root.winfo_screenwidth()
hs = splash_root.winfo_screenheight()
x = (ws - w) // 2
y = (hs - h) // 2
splash_root.geometry(f"{w}x{h}+{x}+{y}")
tk.Label(splash_root, text="⛵ Odysseus", font=("Segoe UI", 22, "bold"), bg="#1a1c23", fg="#e06c75").pack(pady=(22, 2))
tk.Label(splash_root, text="Launching background services...", font=("Segoe UI", 10), bg="#1a1c23", fg="#d1d4e0").pack(pady=2)
tk.Label(splash_root, text="Please wait, this will take a few seconds.", font=("Segoe UI", 8, "italic"), bg="#1a1c23", fg="#5c6370").pack(pady=(12, 0))
splash_root.attributes("-topmost", True)
splash_root.mainloop()
except Exception:
pass
# Launch the GUI splash screen immediately on a background thread
threading.Thread(target=show_splash_instantly, daemon=True).start()
def create_tray_image():
# Generate a beautiful 64x64 icon matching Odysseus brand red accent (#e06c75)
from PIL import Image, ImageDraw
image = Image.new('RGBA', (64, 64), (0, 0, 0, 0))
dc = ImageDraw.Draw(image)
accent_red = (224, 108, 117, 255)
light_red = (224, 108, 117, 150)
# Draw premium sailing boat
dc.polygon([(32, 10), (32, 45), (12, 45)], fill=accent_red)
dc.polygon([(32, 18), (32, 45), (48, 45)], fill=light_red)
dc.polygon([(8, 48), (56, 48), (44, 56), (20, 56)], fill=accent_red)
return image
def on_open_browser(icon, item, url):
webbrowser.open(url)
def on_exit(icon, item):
icon.stop()
os._exit(0)
def setup_system_tray(url):
try:
import pystray
icon_img = create_tray_image()
menu = (
pystray.MenuItem('Open Odysseus', lambda icon, item: on_open_browser(icon, item, url), default=True),
pystray.MenuItem('Exit', on_exit)
)
tray_icon = pystray.Icon(
"Odysseus",
icon_img,
"Odysseus",
menu
)
tray_icon.run()
except Exception:
pass
def open_browser(url):
# Allow uvicorn and app lifecycles to complete warmups
time.sleep(3.5)
# Safely close the splash screen
try:
global splash_root
if splash_root:
splash_root.after(0, splash_root.destroy)
except Exception:
pass
webbrowser.open(url)
if __name__ == "__main__":
import uvicorn
# Import the FastAPI app from app.py
from app import app
bind_host = os.getenv("APP_BIND", "127.0.0.1")
bind_port = int(os.getenv("APP_PORT", "7000"))
url = f"http://{bind_host}:{bind_port}"
if getattr(sys, 'frozen', False):
# Start browser manager thread
threading.Thread(target=open_browser, args=(url,), daemon=True).start()
# Start system tray manager thread
threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start()
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")
licenses/OpenDyslexic-OFL.txt
-94
View File
@@ -1,94 +0,0 @@
Copyright (c) 2019-07-29, Abbie Gonzalez (https://abbiecod.es|support@abbiecod.es),
with Reserved Font Name OpenDyslexic.
Copyright (c) 12/2012 - 2019
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
http://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.
mcp_servers/email_server.py
+16 -275
View File
@@ -23,7 +23,6 @@ import os.path
from pathlib import Path
from datetime import datetime, timedelta
import uuid
from contextvars import ContextVar
from mcp.server import Server
from mcp.server.stdio import stdio_server
@@ -56,13 +55,6 @@ def _uid_fetch_rows(data) -> list:
# flat keys when no DB row matches (legacy single-account behaviour).
_ACCOUNT_CACHE: dict = {} # key = normalized account selector -> config dict
_MCP_OWNER_ARG = "_odysseus_owner"
_CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None)
_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_EMAIL_OWNER", "ODYSSEUS_EMAIL_OWNER")
_OWNER_SCOPE_ERROR = (
"Error: email MCP requires an authenticated owner or ODYSSEUS_MCP_EMAIL_OWNER "
"when owner-scoped email accounts are configured."
)
def _clean_header_value(value) -> str:
@@ -76,59 +68,6 @@ def _db_path() -> Path:
return Path(APP_DB)
def _configured_owner() -> str | None:
for key in _OWNER_ENV_KEYS:
owner = os.environ.get(key, "").strip()
if owner:
return owner
return None
def _current_owner() -> str:
owner = _CURRENT_OWNER.get()
return str(owner or _configured_owner() or "").strip()
def _account_owner(row: dict) -> str:
return str(row.get("owner") or "").strip()
def _has_owner_scoped_accounts(rows: list[dict]) -> bool:
return any(_account_owner(r) for r in rows)
def _account_visible_to_owner(row: dict, owner: str) -> bool:
row_owner = _account_owner(row)
if row_owner == owner:
return True
if row_owner:
return False
# Legacy ownerless accounts are only visible to a scoped caller when the
# mailbox itself matches the owner, mirroring the HTTP email route fallback.
owner_l = owner.lower()
return owner_l in {
str(row.get("imap_user") or "").strip().lower(),
str(row.get("from_address") or "").strip().lower(),
}
def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]:
owner = _current_owner()
if owner:
return [r for r in rows if _account_visible_to_owner(r, owner)]
if _has_owner_scoped_accounts(rows):
return []
return rows
def _mcp_owner_required(rows: list[dict] | None = None) -> bool:
if _current_owner():
return False
rows = rows if rows is not None else _read_accounts_from_db()
return _has_owner_scoped_accounts(rows)
def _load_email_writing_style() -> str:
"""Return the existing Settings > Email > Writing Style value."""
try:
@@ -182,8 +121,9 @@ def _default_document_owner() -> str | None:
return None
def _read_accounts_from_db() -> list:
"""Return all enabled email account rows. Empty list if missing. Never raises."""
def _list_accounts_raw() -> list:
"""Return list of dicts from the email_accounts table. Empty list if table
missing or empty. Never raises."""
path = _db_path()
if not path.exists():
return []
@@ -191,10 +131,9 @@ def _read_accounts_from_db() -> list:
conn = sqlite3.connect(str(path))
conn.row_factory = sqlite3.Row
columns = {r[1] for r in conn.execute("PRAGMA table_info(email_accounts)").fetchall()}
owner_select = "owner" if "owner" in columns else "NULL AS owner"
smtp_security_select = "smtp_security" if "smtp_security" in columns else "'' AS smtp_security"
rows = conn.execute(f"""
SELECT id, {owner_select}, name, is_default, enabled,
SELECT id, name, is_default, enabled,
imap_host, imap_port, imap_user, imap_password, imap_starttls,
smtp_host, smtp_port, {smtp_security_select}, smtp_user, smtp_password, from_address
FROM email_accounts WHERE enabled = 1
@@ -208,15 +147,11 @@ def _read_accounts_from_db() -> list:
return []
def _list_accounts_raw() -> list:
"""Return owner-visible email account rows for the active MCP call."""
return _filter_accounts_for_owner(_read_accounts_from_db())
def _resolve_account_from_rows(rows: list[dict], selector: str | None) -> dict | None:
def _resolve_account(selector: str | None) -> dict | None:
"""Given a selector (None = default, or a name/user/id string), return the
matching row or None. Matching is case-insensitive substring on name +
imap_user + from_address, plus exact id match."""
rows = _list_accounts_raw()
if not rows:
return None
if not selector:
@@ -251,10 +186,6 @@ def _resolve_account_from_rows(rows: list[dict], selector: str | None) -> dict |
return None
def _resolve_account(selector: str | None) -> dict | None:
return _resolve_account_from_rows(_list_accounts_raw(), selector)
def _load_config(account: str | None = None) -> dict:
"""Return the full config dict for the requested account (or default).
@@ -263,7 +194,7 @@ def _load_config(account: str | None = None) -> dict:
2. env vars + settings.json flat keys (legacy)
3. hardcoded fallbacks (localhost:31143 etc.)
"""
cache_key = (_current_owner(), (account or "").strip().lower() or "__default__")
cache_key = (account or "").strip().lower() or "__default__"
if cache_key in _ACCOUNT_CACHE:
return _ACCOUNT_CACHE[cache_key]
@@ -292,13 +223,8 @@ def _load_config(account: str | None = None) -> dict:
"account_name": None,
}
raw_rows = _read_accounts_from_db()
if _mcp_owner_required(raw_rows):
raise ValueError(_OWNER_SCOPE_ERROR)
rows = _filter_accounts_for_owner(raw_rows)
row = _resolve_account_from_rows(rows, account)
if _current_owner() and raw_rows and not rows:
raise ValueError("No email account is configured for the authenticated owner")
rows = _list_accounts_raw()
row = _resolve_account(account)
if account and rows and not row:
available = ", ".join(
f"{r.get('name') or r.get('imap_user')} <{r.get('imap_user') or r.get('from_address') or '?'}>"
@@ -559,148 +485,6 @@ def _get_cached_summaries():
return {}
def _fixture_email_file() -> Path:
return DATA_DIR / "fixture_email_messages.json"
def _fixture_email_enabled() -> bool:
return _fixture_email_file().exists()
def _parse_fixture_date(raw_date: str) -> tuple[str, float]:
if not raw_date:
return "", 0.0
parsed = None
try:
parsed = datetime.fromisoformat(str(raw_date).replace("Z", "+00:00"))
except Exception:
try:
parsed = email.utils.parsedate_to_datetime(str(raw_date))
except Exception:
parsed = None
if parsed:
return parsed.isoformat(), parsed.timestamp()
return str(raw_date), 0.0
def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict:
sender = str(row.get("from") or "Fixture Sender <fixture@example.invalid>")
sender_name, sender_addr = email.utils.parseaddr(sender)
date_str, date_epoch = _parse_fixture_date(str(row.get("date") or ""))
subject = str(row.get("subject") or "(no subject)")
body = str(row.get("body") or "")
owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default")
uid = str(uid_num)
return {
"uid": uid,
"message_id": f"<fixture-email-{uid}-{owner_key}@fixtures.odysseus.local>",
"subject": subject,
"from": sender_name or sender_addr or sender,
"from_address": sender_addr,
"date": date_str,
"date_epoch": date_epoch,
"summary": body[:240],
"body": body,
"account": "Fixture Inbox",
"account_email": owner or str(row.get("owner") or ""),
"account_id": "fixture-email",
"attachments": [],
}
def _fixture_email_rows(owner: str | None = None) -> list[dict]:
path = _fixture_email_file()
if not path.exists():
return []
try:
raw = json.loads(path.read_text(encoding="utf-8"))
except Exception:
return []
rows = raw.get("messages") if isinstance(raw, dict) else raw
out = []
owner = str(owner or "").strip()
for i, row in enumerate(rows if isinstance(rows, list) else [], start=1):
if not isinstance(row, dict):
continue
row_owner = str(row.get("owner") or "").strip()
if owner and row_owner and row_owner != owner:
continue
out.append(_fixture_email_record(row, i, owner or row_owner))
out.sort(key=lambda item: item.get("date_epoch") or 0, reverse=True)
return out
def _fixture_account_rows() -> list[dict]:
if not _fixture_email_enabled():
return []
owner = _current_owner()
owners = []
for row in _fixture_email_rows(owner or None):
email_addr = row.get("account_email") or owner or "fixture@fixtures.odysseus.local"
if email_addr not in owners:
owners.append(email_addr)
if not owners:
owners = [owner or "fixture@fixtures.odysseus.local"]
return [
{
"id": "fixture-email",
"owner": owner or owners[0],
"name": "Fixture Inbox",
"is_default": True,
"imap_user": owners[0],
"from_address": owners[0],
}
]
def _fixture_email_matches(item: dict, query: str) -> bool:
if not query:
return True
terms = [term for term in re.split(r"\W+", str(query).lower()) if term]
haystack = "\n".join(
str(item.get(key) or "")
for key in ("subject", "from", "from_address", "body", "summary")
).lower()
return all(term in haystack for term in terms)
def _fixture_list_emails(folder="INBOX", max_results=20, unresponded_only=False,
unread_only=False, account=None) -> list[dict] | None:
if not _fixture_email_enabled():
return None
if account and str(account).strip().lower() not in {
"fixture-email",
"fixture inbox",
"fixture",
str(_current_owner()).lower(),
}:
return []
if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
return []
return _fixture_email_rows(_current_owner())[: int(max_results or 20)]
def _fixture_search_emails(query, folders=None, max_results=20, account=None) -> list[dict] | None:
if not _fixture_email_enabled():
return None
rows = _fixture_list_emails("INBOX", max_results=1000, account=account) or []
out = [dict(row, _folder="INBOX") for row in rows if _fixture_email_matches(row, str(query or ""))]
return out[: int(max_results or 20)]
def _fixture_read_email(uid=None, message_id=None, folder="INBOX", account=None) -> dict | None:
if not _fixture_email_enabled():
return None
if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}:
return {"error": f"Email UID {uid or message_id} not found"}
for item in _fixture_email_rows(_current_owner()):
if uid and str(item.get("uid")) == str(uid):
return item
if message_id and str(item.get("message_id")) == str(message_id):
return item
return {"error": f"Email not found with UID/Message-ID: {uid or message_id}"}
# ── Tool implementations ──
@@ -711,9 +495,6 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False,
Pass unread_only=True and/or unresponded_only=True for attention scans.
account selects mailbox (None = default).
"""
fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, account)
if fixture is not None:
return fixture
conn = None
try:
conn = _imap_connect(account)
@@ -795,9 +576,6 @@ def _result_sort_time(result: dict) -> datetime:
def _list_emails_across_accounts(folder="INBOX", max_results=20,
unresponded_only=False, unread_only=False):
fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, None)
if fixture is not None:
return fixture, []
rows = _list_accounts_raw()
combined = []
errors = []
@@ -831,9 +609,6 @@ def _search_emails(query, folders=None, max_results=20, account=None):
_list_emails plus an `_folder` tag."""
if not query or not str(query).strip():
return []
fixture = _fixture_search_emails(query, folders=folders, max_results=max_results, account=account)
if fixture is not None:
return fixture
q = str(query).replace("\\", "\\\\").replace('"', '\\"')
# Mail clients commonly use OR FROM/SUBJECT/TEXT to match either field.
# IMAP SEARCH OR is binary, so we nest it.
@@ -956,9 +731,6 @@ def _extract_attachment_to_disk(msg, index, target_dir):
def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
"""Read full email content by UID or message-ID. account = mailbox selector."""
fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=account)
if fixture is not None:
return fixture
cfg = _load_config(account)
conn = None
try:
@@ -1012,9 +784,6 @@ def _read_email(uid=None, message_id=None, folder="INBOX", account=None):
def _read_email_across_accounts(uid=None, message_id=None, folder="INBOX"):
fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=None)
if fixture is not None:
return fixture
rows = _list_accounts_raw()
matches = []
errors = []
@@ -1184,7 +953,7 @@ def _stash_agent_draft(*, to, subject, body, in_reply_to=None, references=None,
now,
account or None,
"agent_draft",
_current_owner(),
"",
))
conn.commit()
conn.close()
@@ -1214,14 +983,10 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b
UI. This closes the auto-send hole that let earlier models invent
signatures and ship them to real recipients without confirmation."""
if _read_agent_email_confirm_setting():
# Even confirmation-first sends must resolve the selected account now.
# Otherwise a caller could stage a pending draft against another
# owner's account selector before browser approval handles it.
cfg = _load_config(account)
return _stash_agent_draft(
to=to, subject=subject, body=body,
in_reply_to=in_reply_to, references=references,
cc=cc, bcc=bcc, account=cfg.get("account_id") or account,
cc=cc, bcc=bcc, account=account,
)
send_account, cfg = _resolve_send_config(account)
msg = EmailMessage()
@@ -1374,7 +1139,7 @@ def _create_email_draft_document(
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
doc_title = (title or subject or "Email draft").strip() or "Email draft"
doc_owner = _current_owner() or _default_document_owner()
doc_owner = _default_document_owner()
db = SessionLocal()
try:
@@ -1957,10 +1722,9 @@ async def list_tools() -> list[Tool]:
Tool(
name="reply_to_email",
description=(
"Reply to an existing email by UID. This sends immediately. Do NOT use "
"for normal 'write/draft a reply saying X' requests; use "
"draft_email_reply so the user can review and send from Odysseus. "
"Only use this when the user explicitly says to send now. Automatically threads the reply with "
"Reply to an existing email by UID. This sends immediately; for normal "
"assistant-written replies, prefer draft_email_reply so the user can "
"review and send from Odysseus. Automatically threads the reply with "
"In-Reply-To and References headers, prefixes 'Re:' on the subject, and "
"uses the original sender as the recipient. Set reply_all=true to also CC "
"the original To/Cc recipients. For follow-up 'reply ...' requests, use "
@@ -2161,21 +1925,10 @@ async def list_tools() -> list[Tool]:
@server.call_tool()
async def call_tool(name: str, arguments: dict) -> list[TextContent]:
arguments = dict(arguments) if isinstance(arguments, dict) else {}
owner = str(arguments.pop(_MCP_OWNER_ARG, "") or "").strip()
owner_token = _CURRENT_OWNER.set(owner or None)
try:
all_db_accounts = _read_accounts_from_db()
if _mcp_owner_required(all_db_accounts):
return [TextContent(type="text", text=_OWNER_SCOPE_ERROR)]
if name == "list_email_accounts":
rows = _filter_accounts_for_owner(all_db_accounts)
rows = _list_accounts_raw()
if not rows:
rows = _fixture_account_rows()
if not rows:
if all_db_accounts and owner:
return [TextContent(type="text", text="No email accounts configured for this owner.")]
return [TextContent(type="text", text="No email accounts configured. Legacy single-account mode active.")]
lines = [f"Found {len(rows)} email account(s):\n"]
for r in rows:
@@ -2355,16 +2108,6 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
bcc=arguments.get("bcc"),
account=acct,
)
if "error" in result:
return [TextContent(type="text", text=f"Error: {result['error']}")]
if result.get("pending"):
return [TextContent(
type="text",
text=(
f"Draft staged for approval (pending id: {result.get('pending_id')}). "
"Nothing has been sent yet. Review and approve it in Odysseus before delivery."
),
)]
acct_note = f" (from {result['account']})" if result.get("account") else ""
return [TextContent(type="text", text=f"Sent email to {result['to']} with subject '{result['subject']}'{acct_note}.")]
@@ -2540,8 +2283,6 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
except Exception as e:
return [TextContent(type="text", text=f"Error: {e}")]
finally:
_CURRENT_OWNER.reset(owner_token)
# ── Main ──
mcp_servers/image_gen_server.py
+2 -2
View File
@@ -73,7 +73,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if not model_spec:
for candidate in ("gpt-image-1.5", "gpt-image-1", "dall-e-3"):
try:
await asyncio.to_thread(_resolve_model, candidate)
_resolve_model(candidate)
model_spec = candidate
break
except ValueError:
@@ -81,7 +81,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if not model_spec:
return [TextContent(type="text", text="Error: No image model found. Configure one in Admin.")]
url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec)
url, model_id, headers = _resolve_model(model_spec)
is_gpt_image = "gpt-image" in model_id.lower()
base_url = url.replace("/chat/completions", "").replace("/v1/messages", "").rstrip("/")
mcp_servers/memory_server.py
+28 -89
View File
@@ -6,7 +6,6 @@ Imports MemoryManager and MemoryVectorStore from the Odysseus codebase.
"""
import asyncio
import os
import sys
import time
from pathlib import Path
@@ -24,55 +23,6 @@ _memory_manager = None
_memory_vector = None
_initialized = False
_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER")
_OWNER_SCOPE_ERROR = (
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
)
def _configured_owner() -> str | None:
for key in _OWNER_ENV_KEYS:
owner = os.environ.get(key, "").strip()
if owner:
return owner
return None
def _entry_owner(entry: dict) -> str | None:
owner = entry.get("owner")
if owner is None:
return None
owner_text = str(owner).strip()
return owner_text or None
def _owner_scoped_store(entries: list[dict]) -> bool:
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
"""Return configured owner, all entries, visible entries, and optional error."""
entries = _memory_manager.load_all()
owner = _configured_owner()
if owner is None and _owner_scoped_store(entries):
return None, entries, [], _OWNER_SCOPE_ERROR
if owner is None:
visible = [
entry for entry in entries
if isinstance(entry, dict) and _entry_owner(entry) is None
]
else:
visible = [
entry for entry in entries
if isinstance(entry, dict) and _entry_owner(entry) == owner
]
return owner, entries, visible, None
def _text_result(text: str) -> list[TextContent]:
return [TextContent(type="text", text=text)]
def _ensure_init():
"""Lazy-init memory managers on first use."""
@@ -125,26 +75,24 @@ async def list_tools() -> list[Tool]:
@server.call_tool()
async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if name != "manage_memory":
return _text_result(f"Unknown tool: {name}")
return [TextContent(type="text", text=f"Unknown tool: {name}")]
_ensure_init()
if not _memory_manager:
return _text_result("Error: Memory manager not available")
return [TextContent(type="text", text="Error: Memory manager not available")]
action = arguments.get("action", "")
if action == "list":
category_filter = arguments.get("category", "")
_owner, _all_memories, memories, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
memories = _memory_manager.load()
if category_filter:
memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()]
if not memories:
msg = "No memories found"
if category_filter:
msg += f" in category '{category_filter}'"
return _text_result(msg + ".")
return [TextContent(type="text", text=msg + ".")]
lines = [f"Found {len(memories)} memory entries:\n"]
for m in memories:
@@ -154,17 +102,15 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if len(text) > 150:
text = text[:150] + "..."
lines.append(f"- [{cat}] `{mid}` — {text}")
return _text_result("\n".join(lines))
return [TextContent(type="text", text="\n".join(lines))]
elif action == "add":
text = arguments.get("text", "")
category = arguments.get("category", "fact")
if not text:
return _text_result("Error: Memory text cannot be empty")
owner, memories, _visible, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
return [TextContent(type="text", text="Error: Memory text cannot be empty")]
entry = _memory_manager.add_entry(text, source="ai_agent", category=category)
memories = _memory_manager.load_all()
memories.append(entry)
_memory_manager.save(memories)
if _memory_vector and _memory_vector.healthy:
@@ -172,28 +118,25 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
_memory_vector.add(entry["id"], text)
except Exception:
pass
return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")
return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")]
elif action == "edit":
memory_id = arguments.get("memory_id", "")
new_text = arguments.get("text", "")
if not memory_id or not new_text:
return _text_result("Error: edit needs memory_id and text")
_owner, memories, visible, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
return [TextContent(type="text", text="Error: edit needs memory_id and text")]
memories = _memory_manager.load_all()
found = False
full_id = None
for m in visible:
if m.get("id", "").startswith(memory_id):
full_id = m["id"]
break
if not full_id:
return _text_result(f"Error: Memory '{memory_id}' not found")
for m in memories:
if m.get("id") == full_id:
if m.get("id", "").startswith(memory_id):
m["text"] = new_text
m["timestamp"] = int(time.time())
found = True
full_id = m["id"]
break
if not found:
return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
_memory_manager.save(memories)
if _memory_vector and _memory_vector.healthy and full_id:
try:
@@ -201,26 +144,24 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
_memory_vector.add(full_id, new_text)
except Exception:
pass
return _text_result(f"Memory updated: {new_text}")
return [TextContent(type="text", text=f"Memory updated: {new_text}")]
elif action == "delete":
memory_id = arguments.get("memory_id", "")
if not memory_id:
return _text_result("Error: delete needs memory_id")
_owner, memories, visible, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
return [TextContent(type="text", text="Error: delete needs memory_id")]
memories = _memory_manager.load_all()
full_id = None
deleted_text = ""
deleted_category = ""
for m in visible:
for m in memories:
if m.get("id", "").startswith(memory_id):
full_id = m["id"]
deleted_text = m.get("text", "")
deleted_category = m.get("category", "")
break
if not full_id:
return _text_result(f"Error: Memory '{memory_id}' not found")
return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
memories = [m for m in memories if m.get("id") != full_id]
_memory_manager.save(memories)
if _memory_vector and _memory_vector.healthy and full_id:
@@ -230,32 +171,30 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
pass
cat = f"[{deleted_category}] " if deleted_category else ""
snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..."
return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})")
return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")]
elif action == "search":
query = arguments.get("text", "")
if not query:
return _text_result("Error: search needs text (query)")
_owner, _all_memories, memories, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
return [TextContent(type="text", text="Error: search needs text (query)")]
memories = _memory_manager.load()
if hasattr(_memory_manager, 'get_relevant_memories'):
results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
else:
query_lower = query.lower()
results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
if not results:
return _text_result(f"No memories found matching '{query}'.")
return [TextContent(type="text", text=f"No memories found matching '{query}'.")]
lines = [f"Found {len(results)} matching memories:\n"]
for m in results:
cat = m.get("category", "fact")
mid = m.get("id", "?")[:8]
text = m.get("text", "")
lines.append(f"- [{cat}] `{mid}` — {text}")
return _text_result("\n".join(lines))
return [TextContent(type="text", text="\n".join(lines))]
else:
return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")
return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")]
async def run():
package-lock.json
Generated
+78 -4
View File
@@ -4,19 +4,93 @@
"requires": true,
"packages": {
"": {
"dependencies": {
"@anthropic-ai/sdk": "^0.104.1"
},
"devDependencies": {
"@antithesishq/bombadil": "^0.6.1"
"@antithesishq/bombadil": "^0.5.0"
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.104.1",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.104.1.tgz",
"integrity": "sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==",
"license": "MIT",
"dependencies": {
"json-schema-to-ts": "^3.1.1",
"standardwebhooks": "^1.0.0"
},
"bin": {
"anthropic-ai-sdk": "bin/cli"
},
"peerDependencies": {
"zod": "^3.25.0 || ^4.0.0"
},
"peerDependenciesMeta": {
"zod": {
"optional": true
}
}
},
"node_modules/@antithesishq/bombadil": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",
"integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.5.0.tgz",
"integrity": "sha512-s0zImmr0iyvSP6QcVLvf40CUiZYIdWBAxiq20uhzujwvfitYa3PGJN652k/pLtVccHM/JrGQxZdvLnihZpltHA==",
"dev": true,
"license": "MIT",
"bin": {
"bombadil": "bin/bombadil.js"
}
},
"node_modules/@babel/runtime": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz",
"integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==",
"license": "MIT",
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@stablelib/base64": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz",
"integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==",
"license": "MIT"
},
"node_modules/fast-sha256": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz",
"integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==",
"license": "Unlicense"
},
"node_modules/json-schema-to-ts": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz",
"integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.18.3",
"ts-algebra": "^2.0.0"
},
"engines": {
"node": ">=16"
}
},
"node_modules/standardwebhooks": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz",
"integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==",
"license": "MIT",
"dependencies": {
"@stablelib/base64": "^1.0.0",
"fast-sha256": "^1.3.0"
}
},
"node_modules/ts-algebra": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
"integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
"license": "MIT"
}
}
}
package.json
+4 -1
View File
@@ -4,6 +4,9 @@
"url": "https://github.com/pewdiepie-archdaemon/odysseus.git"
},
"devDependencies": {
"@antithesishq/bombadil": "^0.6.1"
"@antithesishq/bombadil": "^0.5.0"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.104.1"
}
}
requirements.txt
-1
View File
@@ -3,7 +3,6 @@ uvicorn
python-multipart
python-dotenv
httpx
httpcore>=1.0,<2.0
pydantic>=2.13.4
pydantic-settings>=2.14.1
SQLAlchemy
routes/api_token_routes.py
-2
View File
@@ -160,8 +160,6 @@ def setup_api_token_routes() -> APIRouter:
payload = await request.json()
except Exception:
payload = {}
if not isinstance(payload, dict):
payload = {}
with get_db_session() as db:
token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
if not token:
routes/assistant_routes.py
+2 -3
View File
@@ -16,7 +16,6 @@ from pydantic import BaseModel
from core.database import SessionLocal, CrewMember, ScheduledTask
from src.auth_helpers import get_current_user
from core.auth import RESERVED_USERNAMES
from src.task_scheduler import compute_next_run
@@ -90,11 +89,11 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
# check-in tasks seeded. Hitting any /assistant route under one of these
# used to seed a full CrewMember + Morning/Midday/Evening tasks under that
# owner, which then double-fired alongside the real user's check-ins.
# RESERVED_USERNAMES covers the same set; the `not owner` guard handles "".
_SYNTHETIC_OWNERS = frozenset({"internal-tool", "api", "demo", "system", ""})
async def _get_or_create(owner: str) -> CrewMember:
"""Return the per-owner assistant CrewMember, creating it on demand."""
if not owner or owner in RESERVED_USERNAMES:
if not owner or owner in _SYNTHETIC_OWNERS:
raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}")
db = SessionLocal()
try:
routes/auth_routes.py
+11 -45
View File
@@ -12,8 +12,8 @@ import re
from pathlib import Path
from core.atomic_io import atomic_write_json, atomic_write_text
from core.auth import AuthManager, RESERVED_USERNAMES, SetAdminResult, TOKEN_TTL
from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, PASSWORD_MIN_LENGTH, SKILLS_DIR
from core.auth import AuthManager, SetAdminResult
from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, SKILLS_DIR
from src.rate_limiter import RateLimiter
from src.settings_scrub import scrub_settings
from src.settings import (
@@ -102,12 +102,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
raise HTTPException(429, "Too many requests — try again later")
if auth_manager.is_configured:
raise HTTPException(400, "Already configured")
if len(body.password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.username.strip()) < 1:
raise HTTPException(400, "Username is required")
if body.username.lower() in RESERVED_USERNAMES:
raise HTTPException(403, "Username is reserved")
if len(body.password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
ok = await asyncio.to_thread(auth_manager.setup, body.username, body.password)
if not ok:
raise HTTPException(500, "Setup failed")
@@ -122,12 +118,10 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
raise HTTPException(400, "Run setup first")
if not auth_manager.signup_enabled:
raise HTTPException(403, "Registration is disabled. Ask an admin for an account.")
if len(body.password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
if len(body.username.strip()) < 1:
raise HTTPException(400, "Username is required")
if body.username.lower() in RESERVED_USERNAMES:
raise HTTPException(403, "Username is reserved")
ok = await asyncio.to_thread(auth_manager.create_user, body.username, body.password, is_admin=False)
if not ok:
raise HTTPException(409, "Username already taken")
@@ -150,8 +144,6 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
raise HTTPException(401, "Invalid 2FA code")
# All checks passed — create session (password already verified above)
token = await asyncio.to_thread(auth_manager.create_session_trusted, username)
if not token:
raise HTTPException(401, "Invalid credentials")
cookie_kwargs = dict(
key=SESSION_COOKIE,
value=token,
@@ -161,7 +153,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
path="/",
)
if body.remember:
cookie_kwargs["max_age"] = TOKEN_TTL
cookie_kwargs["max_age"] = 60 * 60 * 24 * 7 # 7 days
response.set_cookie(**cookie_kwargs)
return {"ok": True, "username": username}
@@ -190,18 +182,13 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
pass
return result
@router.get("/policy")
async def auth_policy():
"""Return public auth policy constants for the frontend."""
return auth_manager.policy()
@router.post("/change-password")
async def change_password(body: ChangePasswordRequest, request: Request):
user = _get_current_user(request)
if not user:
raise HTTPException(401, "Not authenticated")
if len(body.new_password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.new_password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
current_token = request.cookies.get(SESSION_COOKIE)
ok = await asyncio.to_thread(auth_manager.change_password, user, body.current_password, body.new_password)
if not ok:
@@ -281,12 +268,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
user = _get_current_user(request)
if not user or not auth_manager.is_admin(user):
raise HTTPException(403, "Admin only")
if len(body.password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.username.strip()) < 1:
raise HTTPException(400, "Username is required")
if body.username.lower() in RESERVED_USERNAMES:
raise HTTPException(403, "Username is reserved")
if len(body.password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
ok = auth_manager.create_user(body.username, body.password, body.is_admin)
if not ok:
raise HTTPException(409, "Username already taken")
@@ -449,23 +432,6 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
except Exception as e:
logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e)
# direct personal RAG uploads live in per-owner directories and the
# vector metadata also carries the username used for owner-filtered
# search. Keep both in sync with the auth rename.
try:
from routes.personal_routes import rename_personal_upload_owner
personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None)
if personal_docs_manager is not None:
rag_manager = getattr(personal_docs_manager, "rag_manager", None)
rename_personal_upload_owner(
old_username,
new_username,
personal_docs_manager=personal_docs_manager,
rag_manager=rag_manager,
)
except Exception as e:
logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e)
# skills: SKILL.md frontmatter carries owner: <username>; the usage
# sidecar (_usage.json) keys entries as owner::skill-name. Both must
# be updated or the renamed user's Skills panel goes empty.
routes/calendar_routes.py
+5 -128
View File
@@ -1,7 +1,6 @@
"""Calendar routes — local SQLite-backed calendar CRUD."""
import logging
import json
import re
import uuid
from datetime import datetime, date, timedelta
@@ -13,9 +12,8 @@ from sqlalchemy import or_, and_
from dateutil.rrule import rrulestr
from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
from src.auth_helpers import effective_user, require_user
from src.auth_helpers import require_user
from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
from src.upload_handler import reserve_upload_references
logger = logging.getLogger(__name__)
@@ -36,24 +34,6 @@ def _ics_naive_dtstart(dt):
return datetime(dt.year, dt.month, dt.day)
return dt
def _ensure_positive_duration(start_dt, end_dt, all_day):
"""Clamp an imported event's end so it has a positive duration.
Some .ics exporters write a single-day all-day event with DTEND equal to
DTSTART (treating DTEND as inclusive rather than the RFC 5545 exclusive
bound). Stored verbatim that produces a zero-duration row, which the
list_events overlap filter (dtstart < end AND dtend > start) silently
drops — the event never appears on the calendar even though the web UI
would otherwise show it. Normalize a non-positive end to the same default
span used when DTEND is absent: one day for all-day events, one hour
otherwise.
"""
if end_dt <= start_dt:
return start_dt + (timedelta(days=1) if all_day else timedelta(hours=1))
return end_dt
# Single-user fallback identity. Used only when:
# 1. The app is configured for single-user (no auth middleware), AND
# 2. The request didn't resolve to an authenticated user.
@@ -454,20 +434,6 @@ def _parse_dt(s: str) -> datetime:
if t is not None:
return base.replace(hour=t[0], minute=t[1])
# time-first: "3pm today", "9am tomorrow", "11pm tonight"
# (parity with parse_due_for_user, which handles these via the same form)
m = _re.match(r'^(.+?)\s+(today|tonight|tomorrow|tmrw|yesterday)$', lower)
if m:
time_part, word = m.group(1).strip(), m.group(2)
base = today
if word in ("tomorrow", "tmrw"):
base = today + timedelta(days=1)
elif word == "yesterday":
base = today - timedelta(days=1)
t = _parse_time(time_part)
if t is not None:
return base.replace(hour=t[0], minute=t[1])
# next <weekday> [at] TIME
weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
m = _re.match(r'^next\s+(\w+)(?:\s+at)?\s*(.*)$', lower)
@@ -543,7 +509,6 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
"description": ev.description or "",
"location": ev.location or "",
"rrule": ev.rrule or "",
"recurrence_exdates": _recurrence_exdates(ev),
"calendar": ev.calendar.name if ev.calendar else "",
"calendar_href": ev.calendar_id,
"color": ev.color or (ev.calendar.color if ev.calendar else ""),
@@ -557,28 +522,6 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
_RRULE_EXPANSION_LIMIT = 1000
def _recurrence_exdates(ev: CalendarEvent) -> list[str]:
raw = getattr(ev, "recurrence_exdates", "") or ""
if not raw:
return []
try:
values = json.loads(raw)
except Exception:
return []
if not isinstance(values, list):
return []
return [str(v) for v in values if isinstance(v, str) and v.strip()]
def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
if "::" not in uid:
return ""
suffix = uid.split("::", 1)[1]
if ev.all_day:
return suffix[:10]
return suffix[:16]
def _expand_rrule(
ev: CalendarEvent, start: datetime, end: datetime
) -> List[dict]:
@@ -643,7 +586,6 @@ def _expand_rrule(
results = []
truncated = False
base = _event_to_dict(ev)
exdates = set(_recurrence_exdates(ev))
for occ_start in rule.xafter(expand_start, inc=True):
if occ_start >= end:
@@ -664,13 +606,8 @@ def _expand_rrule(
# Build the compound uid: {base_uid}::{date} or ::{datetime}
if ev.all_day:
occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}"
exdate_key = occ_start.strftime("%Y-%m-%d")
else:
occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%dT%H:%M')}"
exdate_key = occ_start.strftime("%Y-%m-%dT%H:%M")
if exdate_key in exdates:
continue
d = dict(base)
d["uid"] = occ_uid
@@ -698,18 +635,9 @@ def _expand_rrule(
# ── Routes ──
def setup_calendar_routes(upload_handler=None) -> APIRouter:
def setup_calendar_routes() -> APIRouter:
router = APIRouter(prefix="/api/calendar", tags=["calendar"])
def _reserve_calendar_uploads(request: Request, *values) -> None:
missing_id = reserve_upload_references(
upload_handler,
effective_user(request),
*values,
)
if missing_id:
raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
# ── CalDAV multi-account helpers ─────────────────────────────────────────
def _get_caldav_accounts(owner: str) -> list:
@@ -923,24 +851,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
'</d:prop></d:propfind>'
)
try:
# Build an SSL context that trusts the operator's custom CA bundle
# (SSL_CERT_FILE / REQUESTS_CA_BUNDLE) so self-signed CalDAV servers
# pass the pre-flight the same way they pass the real sync.
# trust_env=False is kept to block proxy/auth env leakage; the CA
# bundle is loaded explicitly instead.
import ssl as _ssl
_ssl_ctx = _ssl.create_default_context()
# Disable VERIFY_X509_STRICT so certs without a keyUsage extension
# (common in self-signed setups) are accepted, matching the
# requests/urllib3 behavior used by the CalDAV sync path.
_ssl_ctx.verify_flags &= ~_ssl.VERIFY_X509_STRICT
_ca_bundle = _os.environ.get("SSL_CERT_FILE") or _os.environ.get("REQUESTS_CA_BUNDLE")
if _ca_bundle:
if _os.path.isfile(_ca_bundle):
_ssl_ctx.load_verify_locations(_ca_bundle)
else:
logger.warning("CalDAV test: CA bundle %s not found, using system CAs", _ca_bundle)
async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False, verify=_ssl_ctx) as cx:
async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False) as cx:
r = await cx.request(
"PROPFIND", url,
auth=(user, pw),
@@ -1097,7 +1008,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
@router.post("/events")
async def create_event(request: Request, data: EventCreate):
owner = _require_user(request)
_reserve_calendar_uploads(request, data.color, data.description, data.location)
db = SessionLocal()
try:
cal = None
@@ -1159,7 +1069,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
@router.put("/events/{uid}")
async def update_event(request: Request, uid: str, data: EventUpdate):
owner = _require_user(request)
_reserve_calendar_uploads(request, data.color, data.description, data.location)
try:
base_uid = _resolve_base_uid(uid)
except ValueError as e:
@@ -1209,7 +1118,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
db.close()
@router.delete("/events/{uid}")
async def delete_event(request: Request, uid: str, scope: str = "series"):
async def delete_event(request: Request, uid: str):
owner = _require_user(request)
try:
base_uid = _resolve_base_uid(uid)
@@ -1218,22 +1127,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
db = SessionLocal()
try:
ev = _get_or_404_event(db, base_uid, owner)
is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
is_caldav = ev.calendar and ev.calendar.source == "caldav"
if is_occurrence_delete:
key = _occurrence_exdate_key(uid, ev)
if not key:
raise HTTPException(400, "Invalid recurring occurrence uid")
exdates = _recurrence_exdates(ev)
if key not in exdates:
exdates.append(key)
ev.recurrence_exdates = json.dumps(sorted(exdates))
if is_caldav:
ev.caldav_sync_pending = "update"
db.commit()
if is_caldav:
await _push_caldav_event_after_commit(owner, base_uid, "update")
return {"ok": True, "scope": "occurrence", "exdate": key}
if is_caldav:
_record_caldav_delete_tombstone(db, ev, owner)
db.delete(ev)
@@ -1253,7 +1147,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
@router.post("/calendars")
async def create_calendar(request: Request, name: str = "Imported", color: str = "#5b8abf"):
owner = _require_user(request)
_reserve_calendar_uploads(request, color)
db = SessionLocal()
try:
cal = CalendarCal(
@@ -1276,7 +1169,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
@router.put("/calendars/{cal_id}")
async def update_calendar(request: Request, cal_id: str, name: str = None, color: str = None):
owner = _require_user(request)
_reserve_calendar_uploads(request, color)
db = SessionLocal()
try:
cal = _get_or_404_calendar(db, cal_id, owner)
@@ -1334,7 +1226,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
db.commit()
db.refresh(target_cal)
imported = skipped = repaired = 0
imported = skipped = 0
for comp in cal_data.walk():
if comp.name != "VEVENT":
continue
@@ -1370,18 +1262,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
.first()
)
if existing:
# An import predating the clamp below may have stored
# this same event with a non-positive duration, which
# the list_events overlap filter hides. Re-importing
# lands here and would skip without touching that row,
# so the event would stay invisible. Backfill the clamp
# onto the stored row before skipping it.
fixed_end = _ensure_positive_duration(
existing.dtstart, existing.dtend, bool(existing.all_day)
)
if fixed_end != existing.dtend:
existing.dtend = fixed_end
repaired += 1
skipped += 1
continue
@@ -1415,8 +1295,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
else:
end_dt = start_dt + timedelta(hours=1)
end_dt = _ensure_positive_duration(start_dt, end_dt, all_day)
ev = CalendarEvent(
uid=uid_val,
calendar_id=target_cal.id,
@@ -1437,7 +1315,6 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
"ok": True,
"imported": imported,
"skipped": skipped,
"repaired": repaired,
"calendar": cal_display,
"calendar_id": target_cal.id,
}
routes/chat_helpers.py
+26 -146
View File
@@ -14,57 +14,14 @@ from core.database import Session as DBSession, ModelEndpoint
from src.llm_core import normalize_model_id
from src.endpoint_resolver import normalize_base
from src.context_compactor import maybe_compact, trim_for_context
from src.model_context import estimate_tokens
from src.auth_helpers import effective_user
from src.auth_helpers import get_current_user
from src.prompt_security import untrusted_context_message
from src.attachment_refs import attachment_ref
from routes.prefs_routes import _load_for_user as load_prefs_for_user
from fastapi import HTTPException
logger = logging.getLogger(__name__)
_CASUAL_OPENING_RE = re.compile(
r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
re.IGNORECASE,
)
_CASUAL_BLOCKLIST_RE = re.compile(
r"\b(?:cookbook|serve|serving|launch|start|vllm|sglang|llama\.?cpp|ollama|"
r"download|model|email|document|doc|note|calendar|task|search|web|research|"
r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
re.IGNORECASE,
)
def _is_casual_low_signal(text: str) -> bool:
"""Short greetings/slang should not pull memory, skills, RAG, or docs."""
s = str(text or "").strip()
m = _CASUAL_OPENING_RE.match(s)
if not m:
return False
tail = m.group("tail") or ""
if _CASUAL_BLOCKLIST_RE.search(tail):
return False
tail_words = re.findall(r"[A-Za-z0-9_'-]+", tail)
return len(tail_words) <= 2
# Strong references to in-flight fire-and-forget tasks scheduled from this
# module. asyncio only keeps weak references to tasks created via
# create_task, so without this the GC can collect a task mid-execution and
# the background work (extraction, auto-naming) silently never runs.
# Mirrors WebhookManager._spawn_tracked from src/webhook_manager.py.
_BG_TASKS: set[asyncio.Task] = set()
def _spawn_bg(coro) -> asyncio.Task:
"""Schedule a background task and hold a strong reference until it finishes."""
task = asyncio.create_task(coro)
_BG_TASKS.add(task)
task.add_done_callback(_BG_TASKS.discard)
return task
# ── Data containers ────────────────────────────────────────────────────── #
@@ -101,19 +58,11 @@ class ChatContext:
uprefs: dict
preset: PresetInfo
preprocessed: PreprocessedMessage
context_trimmed: bool = False
context_messages_before_trim: int = 0
context_messages_after_trim: int = 0
context_tokens_before_trim: int = 0
context_tokens_after_trim: int = 0
# Documents auto-created server-side during preprocess (e.g. when an
# attached fillable PDF gets rendered into a markdown editor doc).
# The chat route emits a doc_update SSE event for each before streaming
# begins, so the editor pane switches to the new doc immediately.
auto_opened_docs: list = field(default_factory=list)
# Uploads attached to this user turn, resolved and owner-checked for the
# agent's private context. This is not emitted to the browser.
uploaded_files: list = field(default_factory=list)
# ── Helpers ────────────────────────────────────────────────────────────── #
@@ -129,7 +78,7 @@ def _enforce_chat_privileges(request, sess) -> None:
which means unrestricted allowed_models / zero cap -> no-op for them.
"""
try:
user = effective_user(request)
user = get_current_user(request)
except Exception:
user = None
if not user:
@@ -210,9 +159,17 @@ async def auto_name_session(session_manager, sess):
return
owner = getattr(sess, "owner", None)
t_url, t_model, t_headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=owner
)
t_url, t_model, t_headers = resolve_task_endpoint(owner=owner)
if not t_model:
# If no task/utility model is configured at all, fall back to
# the session's own model so auto-naming still works even on
# minimal setups.
from src.endpoint_resolver import resolve_endpoint
_fallback = resolve_endpoint("default", owner=owner)
if _fallback and _fallback[1]:
t_url, t_model, t_headers = _fallback
else:
t_url, t_model, t_headers = sess.endpoint_url, sess.model, sess.headers
if not t_model:
logger.debug("[auto-name] No model provided, skipping")
return
@@ -376,62 +333,6 @@ async def preprocess(
)
def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[str]) -> list[dict]:
"""Resolve current-turn upload IDs into a small tool-facing manifest.
The chat UI already sends attachment ids, and preprocessing inlines as much
text as fits. Agent mode still needs a discoverable bridge for files whose
content was truncated/omitted or when the model chooses file tools. Only
owner-authorized uploads are included, and paths must remain inside the
configured upload directory.
"""
if not att_ids or not upload_handler or not hasattr(upload_handler, "resolve_upload"):
return []
def _read_file_can_open(path: str) -> bool:
try:
from src.tool_execution import _resolve_tool_path
return _resolve_tool_path(path) == os.path.realpath(path)
except Exception:
return False
manifest: list[dict] = []
for att_id in att_ids:
try:
info = upload_handler.resolve_upload(str(att_id), owner=owner)
except Exception:
logger.debug("Failed to resolve upload %r for agent manifest", att_id, exc_info=True)
continue
if not isinstance(info, dict):
continue
path = info.get("path")
if path:
try:
inside = True
if hasattr(upload_handler, "_inside_upload_dir"):
inside = bool(upload_handler._inside_upload_dir(path))
elif hasattr(upload_handler, "inside_base_dir"):
inside = bool(upload_handler.inside_base_dir(path))
if not inside or not os.path.exists(path) or not _read_file_can_open(path):
path = None
except Exception:
path = None
ref = attachment_ref({**info, "id": info.get("id") or str(att_id)})
ref.update({
"id": ref["attachment_id"],
"uri": f"odysseus://attachment/{ref['attachment_id']}",
"read_policy": "owner_checked_upload",
# Transitional compatibility: existing built-in tools can still use
# this path, but only after owner, upload-root, and tool-root checks.
"path": path,
})
manifest.append(ref)
return manifest
def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
"""Add user message to session history and update session name.
In incognito mode, still add to in-memory history (for conversation context)
@@ -445,11 +346,11 @@ def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, inco
def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False):
"""Fire webhook and event_bus events for a new user message."""
if webhook_manager and not compare_mode:
webhook_manager.fire_and_forget("chat.message", {
asyncio.create_task(webhook_manager.fire("chat.message", {
"session_id": session_id, "model": sess.model, "message": message[:2000],
})
}))
from src.event_bus import fire_event
user = effective_user(request)
user = get_current_user(request)
fire_event("message_sent", user)
@@ -675,16 +576,9 @@ async def build_chat_context(
if not incognito:
fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
# Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
# bearer-token chat requests use the token owner instead of the "api" sentinel.
user = effective_user(request)
# Resolve user prefs
user = get_current_user(request)
uprefs = load_prefs_for_user(user)
uploaded_files = build_uploaded_file_manifest(
att_ids or [],
getattr(chat_handler, "upload_handler", None),
getattr(sess, "owner", None),
)
casual_low_signal = _is_casual_low_signal(message)
# Memory enabled?
mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
@@ -694,9 +588,6 @@ async def build_chat_context(
if not allow_tool_preprocessing:
mem_enabled = False
skills_enabled = False
if casual_low_signal:
mem_enabled = False
skills_enabled = False
logger.debug(
"Memory enabled=%s for user=%s (incognito=%s, no_memory=%s, pref=%s)",
mem_enabled, user, incognito, no_memory, uprefs.get("memory_enabled", "NOT_SET"),
@@ -712,11 +603,11 @@ async def build_chat_context(
# Use RAG?
use_rag_val = (str(use_rag).lower() != "false") if use_rag is not None else True
if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
if incognito or not allow_tool_preprocessing or is_research_spinoff:
use_rag_val = False
# If pre-fetched search context was provided (compare mode), skip live web search
skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
skip_web = bool(search_context) or not allow_tool_preprocessing
# Build context preface
# The stream path uses enhanced_message (with CoT/preprocessing applied),
@@ -735,7 +626,7 @@ async def build_chat_context(
incognito=incognito,
use_skills=skills_enabled,
)
if use_rag is not None or is_research_spinoff or casual_low_signal:
if use_rag is not None or is_research_spinoff:
_preface_kwargs["use_rag"] = use_rag_val
preface, rag_sources, web_sources = chat_processor.build_context_preface(**_preface_kwargs)
@@ -743,7 +634,7 @@ async def build_chat_context(
used_memories = getattr(chat_processor, '_last_used_memories', [])
# Inject pre-fetched search context (compare mode)
if search_context and allow_tool_preprocessing and not casual_low_signal:
if search_context and allow_tool_preprocessing:
preface.append(untrusted_context_message("prefetched search context", search_context))
# YouTube transcripts
@@ -787,12 +678,7 @@ async def build_chat_context(
messages, context_length, was_compacted = await maybe_compact(
sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
)
_before_trim_messages = len(messages)
_before_trim_tokens = estimate_tokens(messages)
messages = trim_for_context(messages, context_length)
_after_trim_messages = len(messages)
_after_trim_tokens = estimate_tokens(messages)
_context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
return ChatContext(
preface=preface,
@@ -806,13 +692,7 @@ async def build_chat_context(
uprefs=uprefs,
preset=preset,
preprocessed=preprocessed,
context_trimmed=_context_trimmed,
context_messages_before_trim=_before_trim_messages,
context_messages_after_trim=_after_trim_messages,
context_tokens_before_trim=_before_trim_tokens,
context_tokens_after_trim=_after_trim_tokens,
auto_opened_docs=auto_opened_docs,
uploaded_files=uploaded_files,
)
@@ -1232,7 +1112,7 @@ def run_post_response_tasks(
)))
if _extraction_jobs:
_spawn_bg(_run_extraction_jobs_sequentially(session_id, _extraction_jobs))
asyncio.create_task(_run_extraction_jobs_sequentially(session_id, _extraction_jobs))
# Token accumulation
if last_metrics:
@@ -1240,11 +1120,11 @@ def run_post_response_tasks(
# Webhook
if webhook_manager and not compare_mode:
webhook_manager.fire_and_forget("chat.completed", {
asyncio.create_task(webhook_manager.fire("chat.completed", {
"session_id": session_id, "model": sess.model,
"user_message": message, "response": full_response[:2000],
})
}))
# Auto-name
if needs_auto_name(sess.name):
_spawn_bg(auto_name_session(session_manager, sess))
asyncio.create_task(auto_name_session(session_manager, sess))
routes/chat_routes.py
+43 -208
View File
@@ -3,7 +3,6 @@
import asyncio
import json
import os
import re
import time
import logging
from datetime import datetime
@@ -24,13 +23,12 @@ from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_
from src.session_search import search_session_messages
from src.prompt_security import untrusted_context_message
from core.exceptions import SessionNotFoundError
from src.auth_helpers import effective_user, get_current_user
from src.auth_helpers import get_current_user
from routes.session_routes import _verify_session_owner
from routes.document_helpers import _owner_session_filter
from core.database import SessionLocal, get_session_mode, set_session_mode
from core.database import Session as DBSession, ChatMessage as DBChatMessage
from core.database import Document as DBDocument, ModelEndpoint
from core.log_safety import redact_url
from routes.research_routes import _resolve_research_endpoint
from routes.model_routes import _visible_models
from routes.chat_helpers import (
@@ -41,13 +39,8 @@ from routes.chat_helpers import (
clean_thinking_for_save,
_enforce_chat_privileges,
)
from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent
from src.tool_policy import (
WEB_TOOL_NAMES,
build_effective_tool_policy,
is_web_search_explicitly_denied,
web_search_enabled_for_turn,
)
from src.action_intents import classify_tool_intent as _classify_tool_intent
from src.tool_policy import build_effective_tool_policy
logger = logging.getLogger(__name__)
@@ -69,78 +62,6 @@ def _stream_set(session_id: str, **fields) -> None:
rec.update(fields)
def _message_plain_text(content: Any) -> str:
if isinstance(content, list):
parts: List[str] = []
for block in content:
if isinstance(block, dict):
text = block.get("text")
if isinstance(text, str):
parts.append(text)
elif isinstance(block, str):
parts.append(block)
return " ".join(parts)
return str(content or "")
def _last_user_plain_text(messages: List[Dict[str, Any]]) -> str:
for msg in reversed(messages or []):
if msg.get("role") == "user":
return _message_plain_text(msg.get("content"))
return ""
def _ensure_current_request_is_latest_user(messages: List[Dict[str, Any]], current_message: str) -> List[Dict[str, Any]]:
"""Defensively keep detached streams grounded on the request that created them."""
current = str(current_message or "").strip()
if not current:
return messages
latest = _last_user_plain_text(messages).strip()
if latest == current or current in latest or latest in current:
return messages
logger.warning(
"[chat_stream] latest user context mismatch; appending current request for model call. latest=%r current=%r",
latest[:120],
current[:120],
)
repaired = list(messages or [])
repaired.append({"role": "user", "content": current})
return repaired
_WEB_FOLLOWUP_RE = re.compile(
r"^\s*(?:(?:can|could|would|will)\s+you\s+)?"
r"(?:check|try\s+again|look(?:\s+now|\s+it\s+up)?|search(?:\s+now|\s+online|\s+it)?|"
r"do\s+it|again)\??\s*$",
re.I,
)
_RECENT_WEB_CONTEXT_RE = re.compile(
r"\b(?:weather|forecast|rain|raining|hourly|news|headlines|rate|exchange|currency|"
r"price|current|latest|search|look\s+up|online)\b",
re.I,
)
def _recent_session_text(sess, limit: int = 8, max_chars: int = 2000) -> str:
history = getattr(sess, "history", None) or getattr(sess, "_history", None) or []
chunks: List[str] = []
for msg in history[-limit:]:
content = getattr(msg, "content", None)
if content is None and isinstance(msg, dict):
content = msg.get("content")
text = _message_plain_text(content).strip()
if text:
chunks.append(text)
return " ".join(chunks)[-max_chars:]
def _is_contextual_web_followup(message: str, sess) -> bool:
"""Treat short retry/check replies as web lookups when recent context was web."""
if not message or not _WEB_FOLLOWUP_RE.search(message):
return False
return bool(_RECENT_WEB_CONTEXT_RE.search(_recent_session_text(sess)))
def _resolve_request_workspace(request, raw_value) -> tuple:
"""Resolve the posted workspace for this request: (workspace, rejected).
@@ -205,8 +126,7 @@ def _clear_orphaned_session_endpoint(sess, owner: str | None = None) -> bool:
sess.model = ""
sess.headers = {}
return True
except Exception as e:
logger.warning("Failed to clear orphaned session endpoint", exc_info=e)
except Exception:
db.rollback()
return False
finally:
@@ -224,8 +144,7 @@ def _endpoint_cache_contains_model(endpoint, model: str) -> bool:
return True
try:
models = json.loads(raw) if isinstance(raw, str) else raw
except Exception as e:
logger.warning("Failed to parse cached models list, treating as containing model", exc_info=e)
except Exception:
return True
if not isinstance(models, list) or not models:
return True
@@ -317,8 +236,7 @@ def _recover_empty_session_model(sess, session_id: str, owner: str | None = None
is_chatgpt_subscription = False
try:
cached = json.loads(ep.cached_models) if isinstance(ep.cached_models, str) else (ep.cached_models or [])
except Exception as e:
logger.warning("Failed to parse cached_models for endpoint %r", getattr(ep, "id", "?"), exc_info=e)
except Exception:
cached = []
if not cached:
visible = []
@@ -442,7 +360,7 @@ def setup_chat_routes(
sess = session_manager.get_session(session)
except KeyError:
raise HTTPException(404, f"Session '{session}' not found")
owner = effective_user(request)
owner = get_current_user(request)
if _clear_orphaned_session_endpoint(sess, owner=owner):
raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.")
@@ -588,7 +506,6 @@ def setup_chat_routes(
# below). Skill extraction should only learn from real agent sessions,
# not chats we quietly promoted for a notes/calendar intent.
user_requested_agent = (chat_mode == "agent")
_search_enabled = web_search_enabled_for_turn(allow_web_search, use_web)
# Intent auto-escalation: if the user is clearly asking the assistant
# to create a todo, reminder, or calendar event, promote chat → agent
# for this turn so the LLM has access to manage_notes / manage_calendar.
@@ -606,10 +523,6 @@ def setup_chat_routes(
_tool_intent.category,
_tool_intent.reason,
)
elif chat_mode == "chat" and _search_enabled:
chat_mode = "agent"
auto_escalated = True
logger.info("chat→agent auto-escalation: search enabled")
active_doc_id = form_data.get("active_doc_id", "").strip()
logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}")
@@ -687,7 +600,7 @@ def setup_chat_routes(
# but BEFORE loading. Prevents cross-user session hijack.
_verify_session_owner(request, session)
sess = session_manager.get_session(session)
owner = effective_user(request)
owner = get_current_user(request)
if _clear_orphaned_session_endpoint(sess, owner=owner):
raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.")
# Issue #587: picker shows a model from the endpoint cache but
@@ -702,20 +615,6 @@ def setup_chat_routes(
400,
"No model selected for this chat. Open the model picker and choose one before sending.",
)
if (
chat_mode == "chat"
and isinstance(message, str)
and (not _tool_intent or not _tool_intent.needs_tools)
and _is_contextual_web_followup(message, sess)
):
_tool_intent = ToolIntent(True, "web", "contextual web lookup follow-up")
chat_mode = "agent"
auto_escalated = True
logger.info(
"chat→agent auto-escalation: category=%s reason=%s",
_tool_intent.category,
_tool_intent.reason,
)
except SessionNotFoundError as e:
raise HTTPException(404, str(e))
except (ValueError, ValidationError):
@@ -732,7 +631,7 @@ def setup_chat_routes(
_enforce_chat_privileges(request, sess)
# Ensure session has auth headers
resolve_session_auth(sess, session, owner=effective_user(request))
resolve_session_auth(sess, session, owner=get_current_user(request))
# Check for research_pending BEFORE mode persist overwrites it
do_research = str(use_research).lower() == "true"
@@ -747,8 +646,8 @@ def setup_chat_routes(
elif attachments:
try:
att_ids = [str(x) for x in json.loads(attachments)]
except Exception as e:
logger.warning("Failed to parse attachments JSON, ignoring attachments", exc_info=e)
except Exception:
pass
no_memory = str(form_data.get("no_memory", "")).lower() == "true"
pre_context_tool_policy = build_effective_tool_policy(
@@ -826,15 +725,6 @@ def setup_chat_routes(
logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}")
else:
logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}")
if not active_doc:
_email_doc_q = _doc_db.query(DBDocument).filter(
DBDocument.session_id == session,
DBDocument.is_active == True,
DBDocument.language == "email",
)
active_doc = _owner_session_filter(_email_doc_q, ctx.user).order_by(DBDocument.updated_at.desc()).first()
if active_doc:
logger.info(f"[doc-inject] found email draft by session fallback: title={active_doc.title!r}")
if not active_doc:
_session_doc_q = _doc_db.query(DBDocument).filter(
DBDocument.session_id == session,
@@ -872,35 +762,20 @@ def setup_chat_routes(
# Build disabled-tools set from frontend toggles + user privileges
disabled_tools = set()
# Only disable bash when the caller *explicitly* set it to a falsy
# value. When unset (None), defer to per-user privilege checks below.
# Web search is per-turn opt-in: either the chat pre-search setting
# (`use_web=true`) or agent web toggle (`allow_web_search=true`) must
# explicitly enable it.
# Only disable bash/web_search when the caller *explicitly* set them
# to a falsy value. When unset (None), defer to per-user privilege
# checks below — this lets admins with can_use_bash=True use bash
# by default without having to send allow_bash in every request.
if allow_bash is not None and str(allow_bash).lower() != "true":
disabled_tools.add("bash")
_explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web")
if is_web_search_explicitly_denied(allow_web_search) or not _search_enabled:
disabled_tools.update(WEB_TOOL_NAMES)
if _explicit_web_intent:
# A direct lookup/search request should not drift into personal
# tools or shell fallbacks. It can only use web_search/web_fetch
# when the request's explicit web setting enabled them.
disabled_tools.update({
"bash", "python",
"search_chats", "manage_skills", "manage_memory",
"read_file", "write_file", "edit_file",
"create_document", "edit_document", "update_document",
"send_email", "reply_to_email",
"manage_notes", "manage_calendar", "manage_tasks",
"api_call", "builtin_browser",
})
if _search_enabled:
disabled_tools.difference_update(WEB_TOOL_NAMES)
else:
disabled_tools.update(WEB_TOOL_NAMES)
elif _search_enabled:
disabled_tools.difference_update(WEB_TOOL_NAMES)
if (
allow_web_search is not None
and str(allow_web_search).lower() != "true"
and not _explicit_web_intent
):
disabled_tools.add("web_search")
disabled_tools.add("web_fetch")
# Nobody/incognito mode: deny tools that would expose the user's
# persistent memory, past chats, or other identity-linked data.
@@ -911,19 +786,19 @@ def setup_chat_routes(
"manage_skills", # skill presets tied to user
})
# Active email reader open → strip the tools that let the agent drift
# away from the visible email or skip review. The only allowed compose
# path is ui_control open_email_reply, which opens the same draft editor
# as the Reply button with the generated body pre-filled. This prevents
# the model from falling back to direct SMTP when it botches a draft
# call, and prevents fake email-shaped documents.
# Active email reader open → strip the tools that let the agent
# "drift" to a new compose: create_document (writes a fake email-
# shaped .md file) and send_email (sends fresh to a recipient the
# agent invented). With those gone, the only paths left for "write
# email saying X" are ui_control open_email_reply (draft) and
# reply_to_email (immediate send) — both of which use the open
# email's UID. Code-level enforcement instead of relying on a
# prompt rule the model can ignore.
if active_email_ctx and active_email_ctx.get("uid"):
disabled_tools.update({
"create_document",
"send_email",
"reply_to_email",
"mcp__email__send_email",
"mcp__email__reply_to_email",
})
# Enforce per-user privileges
@@ -1048,7 +923,7 @@ def setup_chat_routes(
if effective_do_research:
_r_ep, _r_model, _r_headers = _resolve_research_endpoint(sess)
_auth_keys = list(_r_headers.keys()) if _r_headers else []
logger.info(f"Research endpoint resolved: model={_r_model}, endpoint={redact_url(_r_ep)}, auth_keys={_auth_keys}, sess_headers_keys={list(sess.headers.keys()) if isinstance(sess.headers, dict) else type(sess.headers)}")
logger.info(f"Research endpoint resolved: model={_r_model}, endpoint={_r_ep}, auth_keys={_auth_keys}, sess_headers_keys={list(sess.headers.keys()) if isinstance(sess.headers, dict) else type(sess.headers)}")
# Clarification round: only for very short/vague queries on first research message.
# Skip in compare mode — each pane is a fresh session, so every one would
@@ -1165,16 +1040,13 @@ def setup_chat_routes(
_active_streams.pop(session, None)
return
messages = _ensure_current_request_is_latest_user(ctx.messages, message)
messages = ctx.messages
# Auto-compact notification
if ctx.was_compacted:
yield f"data: {json.dumps({'type': 'compacted', 'context_length': ctx.context_length})}\n\n"
if ctx.context_trimmed and not ctx.was_compacted:
yield f"data: {json.dumps({'type': 'context_trimmed', 'data': {'context_length': ctx.context_length, 'messages_before': ctx.context_messages_before_trim, 'messages_after': ctx.context_messages_after_trim, 'tokens_before': ctx.context_tokens_before_trim, 'tokens_after': ctx.context_tokens_after_trim}})}\n\n"
full_response = ""
thinking_response = ""
last_metrics = None
# Configured fallback chain for the default chat model. Tried in
@@ -1264,9 +1136,7 @@ def setup_chat_routes(
# Forward them so the client can show a thinking
# indicator, but don't fold them into the saved
# reply (mirrors the rewrite path below).
if data.get("thinking"):
thinking_response += data["delta"]
else:
if not data.get("thinking"):
full_response += data["delta"]
_stream_set(session, partial=full_response)
yield chunk
@@ -1286,12 +1156,6 @@ def setup_chat_routes(
_reported_model = last_metrics.get("model")
last_metrics["requested_model"] = _requested_model
last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
if ctx.context_trimmed:
last_metrics["context_trimmed"] = True
last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
if ctx.context_length and last_metrics.get("input_tokens"):
pct = min(round((last_metrics["input_tokens"] / ctx.context_length) * 100, 1), 100.0)
last_metrics["context_percent"] = pct
@@ -1334,11 +1198,8 @@ def setup_chat_routes(
}
yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
if full_response:
_metrics_to_save = dict(last_metrics or {})
if thinking_response.strip() and not _metrics_to_save.get("thinking"):
_metrics_to_save["thinking"] = thinking_response.strip()
_saved_id = save_assistant_response(
sess, session_manager, session, full_response, _metrics_to_save,
sess, session_manager, session, full_response, last_metrics,
character_name=ctx.preset.character_name,
web_sources=web_sources,
rag_sources=ctx.rag_sources,
@@ -1351,7 +1212,7 @@ def setup_chat_routes(
yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
run_post_response_tasks(
sess, session_manager, session, message, full_response,
_metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
incognito=incognito, compare_mode=compare_mode,
character_name=ctx.preset.character_name,
owner=_user,
@@ -1386,14 +1247,7 @@ def setup_chat_routes(
try:
from src.settings import get_setting
from src.agent_tools import MAX_AGENT_ROUNDS as _DEFAULT_ROUNDS
# Per-message tool budget from settings; guard defensively in
# case settings.json was hand-edited to a non-numeric value
# (the HTTP admin endpoint validates, but direct edits bypass
# it). 0 = unlimited, matching auth_routes set_settings().
try:
_tool_budget = int(get_setting("agent_max_tool_calls", 0))
except (TypeError, ValueError):
_tool_budget = 0
_tool_budget = int(get_setting("agent_max_tool_calls", 0))
# Per-message round cap from settings; clamp defensively in
# case settings.json was hand-edited to a bad value.
try:
@@ -1402,10 +1256,6 @@ def setup_chat_routes(
_max_rounds = _DEFAULT_ROUNDS
_max_rounds = max(1, min(_max_rounds, 200))
_forced_tools = None
if _search_enabled:
_forced_tools = set(WEB_TOOL_NAMES)
async for chunk in stream_agent_loop(
sess.endpoint_url,
sess.model,
@@ -1427,8 +1277,6 @@ def setup_chat_routes(
plan_mode=plan_mode,
approved_plan=approved_plan or None,
workspace=workspace or None,
forced_tools=_forced_tools,
uploaded_files=ctx.uploaded_files,
):
if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"):
try:
@@ -1437,9 +1285,7 @@ def setup_chat_routes(
# Reasoning tokens arrive flagged thinking:true.
# Forward them for the live indicator, but keep
# them out of the saved reply (same as chat mode).
if data.get("thinking"):
thinking_response += data["delta"]
else:
if not data.get("thinking"):
full_response += data["delta"]
_stream_set(session, partial=full_response)
yield chunk
@@ -1450,7 +1296,7 @@ def setup_chat_routes(
"tool_start", "tool_output", "agent_step",
"doc_stream_open", "doc_stream_delta",
"doc_update", "doc_suggestions", "ui_control",
"rounds_exhausted", "budget_exceeded",
"rounds_exhausted",
"loop_breaker_triggered",
"intent_nudge_exhausted",
"ask_user",
@@ -1479,26 +1325,15 @@ def setup_chat_routes(
_reported_model = last_metrics.get("model")
last_metrics["requested_model"] = last_metrics.get("requested_model") or _requested_model
last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
if ctx.context_trimmed:
last_metrics["context_trimmed"] = True
last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
except json.JSONDecodeError:
yield chunk
elif chunk.startswith("event: "):
yield chunk
elif chunk == "data: [DONE]\n\n":
_has_tool_events = bool((last_metrics or {}).get("tool_events"))
if full_response or _has_tool_events:
_response_to_save = full_response or "Done."
_metrics_to_save = dict(last_metrics or {})
if thinking_response.strip() and not _metrics_to_save.get("thinking"):
_metrics_to_save["thinking"] = thinking_response.strip()
if full_response:
_saved_id = save_assistant_response(
sess, session_manager, session, _response_to_save, _metrics_to_save,
sess, session_manager, session, full_response, last_metrics,
character_name=ctx.preset.character_name,
web_sources=web_sources,
rag_sources=ctx.rag_sources,
@@ -1508,8 +1343,8 @@ def setup_chat_routes(
if _saved_id:
yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
run_post_response_tasks(
sess, session_manager, session, message, _response_to_save,
_metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
sess, session_manager, session, message, full_response,
last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
incognito=incognito, compare_mode=compare_mode,
character_name=ctx.preset.character_name,
agent_rounds=_agent_rounds,
@@ -1649,7 +1484,7 @@ def setup_chat_routes(
if not q or not q.strip():
return []
_user = effective_user(request)
_user = get_current_user(request)
return [
result.to_dict()
for result in search_session_messages(
routes/codex_routes.py
+13 -55
View File
@@ -15,7 +15,6 @@ from typing import Any
from fastapi import APIRouter, BackgroundTasks, Body, HTTPException, Request
from fastapi.responses import StreamingResponse
from core.middleware import require_admin
from src.auth_helpers import require_authenticated_request, require_user
from src.tool_implementations import do_manage_notes
from src.constants import COOKBOOK_STATE_FILE
@@ -47,12 +46,8 @@ def _ssh_prefix_for_task(task: dict) -> tuple[str, str]:
shell metacharacters in ``remoteHost`` is rejected with 400 rather than
injected.
"""
raw_host = task.get("remoteHost")
raw_port = task.get("sshPort")
host_value = str(raw_host).strip() if raw_host is not None else None
port_value = str(raw_port).strip() if raw_port is not None else None
host = validate_remote_host(host_value or None) or ""
ssh_port = validate_ssh_port(port_value or None) or ""
host = validate_remote_host((task.get("remoteHost") or "").strip() or None) or ""
ssh_port = validate_ssh_port((task.get("sshPort") or "").strip() or None) or ""
port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
return host, port_flag
@@ -110,20 +105,6 @@ def _scope_owner_all(request: Request, required: set[str]) -> str:
return require_user(request)
def _require_cookbook_scope(request: Request, allowed: set[str]) -> str:
"""Authorize a Codex cookbook route.
For API-token callers, enforce the given scope set.
For cookie-session callers, additionally require admin privileges
because cookbook surfaces expose host topology, task logs, tmux
commands, and model-serving controls.
"""
owner = _scope_owner(request, allowed)
if not getattr(request.state, "api_token", False):
require_admin(request)
return owner
def _find_endpoint(router: APIRouter | None, method: str, path: str):
if router is None:
return None
@@ -133,18 +114,6 @@ def _find_endpoint(router: APIRouter | None, method: str, path: str):
return None
def _clamp_pagination(offset: Any, limit: Any, *, default_limit: int = 50, max_limit: int = 50) -> tuple[int, int]:
try:
parsed_offset = int(0 if offset in (None, "") else offset)
except (TypeError, ValueError):
raise HTTPException(400, "Invalid offset")
try:
parsed_limit = int(default_limit if limit in (None, "") else limit)
except (TypeError, ValueError):
raise HTTPException(400, "Invalid limit")
return max(0, parsed_offset), max(1, min(parsed_limit, max_limit))
def setup_codex_routes(
email_router: APIRouter | None = None,
memory_router: APIRouter | None = None,
@@ -337,10 +306,7 @@ def setup_codex_routes(
@router.post("/emails/draft-document")
async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES)
if docs_owner != owner:
raise HTTPException(403, "API token owner mismatch")
owner = _scope_owner_all(request, {"email:draft", "documents:write"})
if documents_create_endpoint is None:
raise HTTPException(503, "Documents integration is not available")
from routes.document_routes import DocumentCreate
@@ -452,18 +418,10 @@ def setup_codex_routes(
owner = _scope_owner(request, DOCS_READ_SCOPES)
if documents_library_endpoint is None:
raise HTTPException(503, "Documents integration is not available")
offset, limit = _clamp_pagination(offset, limit)
result = await _as_owner(
return await _as_owner(
request, owner, documents_library_endpoint,
request, search, language, sort, offset, limit, archived,
)
if isinstance(result, dict):
docs = result.get("documents")
total = result.get("total")
if isinstance(docs, list) and isinstance(total, int):
next_offset = offset + len(docs)
result["next_offset"] = next_offset if next_offset < total else None
return result
@router.get("/documents/{doc_id}")
async def codex_documents_get(request: Request, doc_id: str):
@@ -567,14 +525,14 @@ def setup_codex_routes(
@router.get("/cookbook/tasks")
async def codex_cookbook_tasks(request: Request):
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
_scope_owner(request, COOKBOOK_READ_SCOPES)
state = _read_cookbook_state()
tasks = state.get("tasks") or []
return {"tasks": [_redact_task(t) for t in tasks]}
@router.get("/cookbook/servers")
async def codex_cookbook_servers(request: Request):
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
_scope_owner(request, COOKBOOK_READ_SCOPES)
state = _read_cookbook_state()
servers = state.get("env", {}).get("servers") or []
# Strip ssh creds / passwords; keep only what's needed to pick a host.
@@ -593,7 +551,7 @@ def setup_codex_routes(
@router.get("/cookbook/output/{session_id}")
async def codex_cookbook_output(request: Request, session_id: str, tail: int = 400):
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
_scope_owner(request, COOKBOOK_READ_SCOPES)
# Defensive: session_id must be the tmux-style id we issue
# (`serve-XXXX` / `cookbook-XXXX` / `queue-XXXX`); anything else
# would let the agent run arbitrary `tmux capture-pane` targets.
@@ -635,7 +593,7 @@ def setup_codex_routes(
@router.post("/cookbook/serve")
async def codex_cookbook_serve(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
# Wraps /api/model/serve with the SAME validation the UI uses.
# _validate_serve_cmd (called inside model_serve) rejects shell
# metachars and requires the leading binary to be in the
@@ -674,7 +632,7 @@ def setup_codex_routes(
@router.post("/cookbook/stop/{session_id}")
async def codex_cookbook_stop(request: Request, session_id: str):
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
import re as _re
if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id):
raise HTTPException(400, "Invalid session id")
@@ -694,7 +652,7 @@ def setup_codex_routes(
"""List cached models on a configured server (or local if host is omitted).
Mirrors `list_cached_models` from the chat agent so external agents have
the same inventory view before deciding what to serve/download."""
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
_scope_owner(request, COOKBOOK_READ_SCOPES)
# Hit /api/model/cached internally, with the same modelDirs the chat
# agent's list_cached_models would resolve from cookbook state.
state = _read_cookbook_state()
@@ -756,7 +714,7 @@ def setup_codex_routes(
"""List saved serve presets (model + host + port + launch cmd).
Counterpart to `list_serve_presets`. Use BEFORE composing a `serve`
body — the user's saved preset usually has the working cmd already."""
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
_scope_owner(request, COOKBOOK_READ_SCOPES)
state = _read_cookbook_state()
presets = state.get("presets") or []
out = []
@@ -776,7 +734,7 @@ def setup_codex_routes(
async def codex_cookbook_serve_preset(request: Request, name: str):
"""Launch a saved preset by name. Reuses the working cmd + host the
user already saved, avoiding the cmd-allowlist trial-and-error loop."""
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
import re as _re
if not _re.fullmatch(r"[A-Za-z0-9 _.:@\-]+", name):
raise HTTPException(400, "Invalid preset name")
@@ -828,7 +786,7 @@ def setup_codex_routes(
cookbook tracking. Needed when serve_model rejects a cmd and the
agent falls back to direct ssh — without adoption the session is
invisible to the UI. Body: {tmux_session, model, host?, port?}."""
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
norm = dict(body or {})
sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip()
model = (norm.get("model") or norm.get("repo_id") or "").strip()
routes/contacts/__init__.py
-5
View File
@@ -1,5 +0,0 @@
"""Contacts route domain package (slice 2e, #4082/#4071).
Contains contacts_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/contacts_routes.py re-exports from here.
"""
routes/contacts/contacts_routes.py
-916
View File
@@ -1,916 +0,0 @@
"""
contacts_routes.py
CardDAV contacts integration. Reads from local Radicale, supports
search and adding new contacts.
"""
import re
import logging
import uuid
import json
import csv
import io
import os
import inspect
import httpx
from pathlib import Path
from datetime import datetime
from urllib.parse import urljoin, urlparse, urlunparse
from core.log_safety import redact_url
from fastapi import APIRouter, Query, Depends, Response, HTTPException
from typing import List, Dict, Optional
from core.middleware import require_admin
from src.url_safety import check_outbound_url
logger = logging.getLogger(__name__)
from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
DATA_DIR = Path(_DATA_DIR)
SETTINGS_FILE = Path(_SETTINGS_FILE)
LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
def _load_settings():
if SETTINGS_FILE.exists():
return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
return {}
def _save_settings(settings):
from core.atomic_io import atomic_write_json
atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
def _get_carddav_config():
import os
settings = _load_settings()
password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
if password and "carddav_password" in settings:
from src.secret_storage import decrypt
password = decrypt(password)
return {
"url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
"username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
"password": password,
}
def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
cfg = cfg or _get_carddav_config()
return bool((cfg.get("url") or "").strip())
def _validate_carddav_url(url: str) -> str:
cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
ok, reason = check_outbound_url(
cleaned,
block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
)
if not ok:
raise ValueError(f"Rejected CardDAV URL: {reason}")
return cleaned
def _carddav_base_url(cfg: Dict) -> str:
return _validate_carddav_url(cfg.get("url") or "")
def _normalize_contact(contact: Dict) -> Dict:
emails = []
for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
e = str(e or "").strip()
if e and e not in emails:
emails.append(e)
phones = []
for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
p = str(p or "").strip()
if p and p not in phones:
phones.append(p)
name = str(contact.get("name") or "").strip()
if not name and emails:
name = emails[0].split("@")[0]
address = str(contact.get("address") or "").strip()
return {
"uid": str(contact.get("uid") or uuid.uuid4()),
"name": name,
"emails": emails,
"phones": phones,
"address": address,
}
def _load_local_contacts() -> List[Dict]:
try:
if not LOCAL_CONTACTS_FILE.exists():
return []
data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
rows = data.get("contacts", data) if isinstance(data, dict) else data
return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
except Exception as e:
logger.error(f"Failed to load local contacts: {e}")
return []
def _save_local_contacts(contacts: List[Dict]) -> None:
from core.atomic_io import atomic_write_json
DATA_DIR.mkdir(parents=True, exist_ok=True)
atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
_contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
_contact_cache["fetched_at"] = datetime.utcnow()
# ── vCard parsing ──
def _vunesc(value: str) -> str:
"""Reverse _vesc() — turn escaped vCard text back into the raw value.
Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
if not value:
return value
out = []
i = 0
while i < len(value):
ch = value[i]
if ch == "\\" and i + 1 < len(value):
nxt = value[i + 1]
if nxt in ("n", "N"):
out.append("\n")
elif nxt in (",", ";", "\\"):
out.append(nxt)
else:
out.append(nxt)
i += 2
else:
out.append(ch)
i += 1
return "".join(out)
def _parse_vcards(text: str) -> List[Dict]:
"""Parse a stream of vCards into dicts with name, email, phone."""
# Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
# space or tab is a continuation of the previous logical line. Real
# CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
# PHOTO lines, and splitting on raw newlines without unfolding dropped the
# continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
# email/name.
text = re.sub(r"\r\n[ \t]", "", text or "")
text = re.sub(r"\n[ \t]", "", text)
contacts = []
for block in re.split(r"BEGIN:VCARD", text):
if not block.strip():
continue
contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
for line in block.split("\n"):
line = line.strip()
# Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
# that Apple Contacts / iCloud / many CardDAV servers emit by
# default — without this the property-name checks below miss those
# lines and silently drop the email / phone. The group token only
# precedes the property name, so it is safe to strip for matching
# and value extraction, and a no-op for non-grouped lines.
name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
if name_part.startswith("FN:") or name_part.startswith("FN;"):
contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
elif name_part.startswith("EMAIL"):
# Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
if ":" in name_part:
email_addr = _vunesc(name_part.split(":", 1)[1])
if email_addr and email_addr not in contact["emails"]:
contact["emails"].append(email_addr)
elif name_part.startswith("TEL"):
if ":" in name_part:
phone = _vunesc(name_part.split(":", 1)[1])
if phone and phone not in contact["phones"]:
contact["phones"].append(phone)
elif name_part.startswith("ADR"):
# vCard ADR is 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
# Recover a human-readable string by joining non-empty
# components with ", ".
if ":" in name_part:
raw = name_part.split(":", 1)[1]
parts = [_vunesc(p).strip() for p in raw.split(";")]
contact["address"] = ", ".join(p for p in parts if p)
elif name_part.startswith("UID:"):
contact["uid"] = _vunesc(name_part[4:])
if contact["name"] or contact["emails"]:
contacts.append(contact)
return contacts
def _vesc(value: str) -> str:
"""Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
or any value containing a newline produces a malformed vCard (broken
N/FN fields) or could inject arbitrary properties."""
return (
(value or "")
.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "")
.replace(",", "\\,")
.replace(";", "\\;")
)
def _build_vcard(name: str, email: str, uid: Optional[str] = None,
emails: Optional[List[str]] = None,
phones: Optional[List[str]] = None,
address: Optional[str] = None) -> str:
"""Build a vCard. Accepts either a single `email` (legacy callers) or
full `emails`/`phones` lists (edit path). The first email is marked
PREF=1. All values are RFC-6350-escaped."""
if not uid:
uid = str(uuid.uuid4())
# Normalize email lists — `email` arg is a convenience for single-email
# creation; `emails` (if given) is authoritative.
email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
# Try to split name into first/last
parts = name.strip().split()
if len(parts) >= 2:
first = parts[0]
last = " ".join(parts[1:])
else:
first = name
last = ""
# N field is structured (5 components separated by ';') — escape each
# component individually so a comma in the name doesn't split it.
n_field = f"{_vesc(last)};{_vesc(first)};;;"
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"UID:{_vesc(uid)}",
f"FN:{_vesc(name)}",
f"N:{n_field}",
]
for i, em in enumerate(email_list):
# First email is the preferred one.
lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
for ph in phone_list:
lines.append(f"TEL:{_vesc(ph)}")
# Address: stuff the whole human-readable string into the street
# component of ADR. vCard ADR has 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
addr = (address or "").strip()
if addr:
lines.append(f"ADR:;;{_vesc(addr)};;;;")
lines.append("END:VCARD")
return "\r\n".join(lines) + "\r\n"
# ── In-memory cache ──
_contact_cache = {"contacts": [], "fetched_at": None}
def _abs_url(href: str) -> str:
"""Combine a multistatus <href> (an absolute path like
/user/contacts/x.vcf) with the configured CardDAV server origin so we
get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
for the configured origin; a cross-origin href is treated as a path on the
configured server so a malicious CardDAV response cannot redirect later
writes/deletes to cloud metadata or another host."""
cfg = _get_carddav_config()
base = _carddav_base_url(cfg)
base_p = urlparse(base)
joined = urljoin(base.rstrip("/") + "/", href or "")
joined_p = urlparse(joined)
if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
return _validate_carddav_url(joined)
# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
# alongside the resource href. Lets us map each contact's UID to the real
# server resource path (which is NOT always <uid>.vcf for contacts created
# by other clients).
_ADDRESSBOOK_QUERY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
'<D:prop><D:getetag/><C:address-data/></D:prop>'
'<C:filter/>'
'</C:addressbook-query>'
)
def _fetch_via_report(cfg, auth):
"""Try a CardDAV REPORT addressbook-query — returns contacts WITH an
`href` field, or None if the server doesn't support it / errors."""
from defusedxml import ElementTree as ET
try:
r = httpx.request(
"REPORT", cfg["url"],
content=_ADDRESSBOOK_QUERY.encode("utf-8"),
headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
auth=auth, timeout=10,
)
if r.status_code not in (207, 200):
return None
root = ET.fromstring(r.text)
ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
out = []
for resp in root.findall("D:response", ns):
href_el = resp.find("D:href", ns)
data_el = resp.find(".//C:address-data", ns)
if href_el is None or data_el is None or not (data_el.text or "").strip():
continue
parsed = _parse_vcards(data_el.text)
if not parsed:
continue
c = parsed[0]
c["href"] = href_el.text.strip()
out.append(c)
# If the REPORT parsed to ZERO contacts, don't trust it — some
# CardDAV servers treat an empty <filter/> as "match nothing" and
# return a valid-but-empty 207. Return None so the caller falls
# back to the plain GET (which lists everything). A genuinely empty
# address book just costs one extra GET that also returns nothing.
if not out:
return None
return out
except Exception as e:
logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
return None
def _fetch_contacts(force=False):
"""Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
if not force and _contact_cache["fetched_at"]:
age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
if age < 60:
return _contact_cache["contacts"]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
try:
cfg["url"] = _carddav_base_url(cfg)
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
# Preferred path: REPORT gives us hrefs for reliable edit/delete.
contacts = _fetch_via_report(cfg, auth)
if contacts is None:
# Fallback: plain GET, concatenated vCards, no hrefs.
r = httpx.get(cfg["url"], auth=auth, timeout=10)
if r.status_code != 200:
logger.warning(f"CardDAV returned {r.status_code}")
return _contact_cache["contacts"]
contacts = _parse_vcards(r.text)
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
except Exception as e:
logger.error(f"Failed to fetch contacts: {e}")
return _contact_cache["contacts"]
def _resolve_resource_url(uid: str) -> str:
"""Map a contact UID to its real CardDAV resource URL. Uses the href
captured during fetch when available (handles contacts whose filename
!= UID); falls back to the <uid>.vcf guess for app-created contacts or
when no href is known."""
def _lookup():
for c in _contact_cache.get("contacts", []):
if c.get("uid") == uid and c.get("href"):
return _abs_url(c["href"])
return None
found = _lookup()
if found:
return found
# Not in cache (or no href) — refresh once and retry before guessing.
try:
_fetch_contacts(force=True)
except Exception:
pass
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
"""Add a new contact via CardDAV or local contacts."""
email = (email or "").strip()
phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = email.lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
return True
contacts.append(_normalize_contact({
"name": name,
"emails": [email] if email else [],
"phones": phone_list,
"address": address,
}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
try:
url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
# Invalidate cache
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to create contact: {e}")
return False
def _vcard_url(uid: str) -> str:
"""The CardDAV resource URL for a given contact UID. The uid is URL-
encoded so a value containing '/', '..' or other path chars can't
escape the collection and target an arbitrary CardDAV resource."""
from urllib.parse import quote
cfg = _get_carddav_config()
return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
def _import_vcards(text: str) -> Dict:
"""Import a (possibly multi-card) .vcf blob. Each card is PUT to the
CardDAV server PRESERVING its full original content (ADR/ORG/photo/
etc.) — we don't rebuild it, just ensure it has VERSION + UID and
normalize line endings. Returns {imported, failed, total}."""
from urllib.parse import quote
cfg = _get_carddav_config()
if not cfg.get("url"):
parsed = _parse_vcards(text)
contacts = _load_local_contacts()
existing = {
e.lower()
for c in contacts
for e in (c.get("emails") or [])
if e
}
imported = 0
for c in parsed:
emails = [e for e in (c.get("emails") or []) if e]
if emails and any(e.lower() in existing for e in emails):
continue
contacts.append(_normalize_contact(c))
for e in emails:
existing.add(e.lower())
imported += 1
if imported:
_save_local_contacts(contacts)
return {"imported": imported, "failed": 0, "total": len(parsed)}
try:
base_url = _carddav_base_url(cfg)
except ValueError as e:
logger.warning("CardDAV import URL rejected: %s", e)
return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
# Split into individual cards. re.split drops the BEGIN line, so we
# re-add it. Normalize CRLF.
raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
blocks = []
for chunk in raw.split("BEGIN:VCARD"):
chunk = chunk.strip()
if not chunk:
continue
# Trim anything after END:VCARD (defensive).
end = chunk.upper().find("END:VCARD")
body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
blocks.append("BEGIN:VCARD\n" + body)
imported = 0
failed = 0
for block in blocks:
# Extract or assign a UID.
m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
if not m:
# Inject a UID right after the VERSION line (or after BEGIN).
if re.search(r"^VERSION:", block, re.MULTILINE):
block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
else:
block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
elif not re.search(r"^VERSION:", block, re.MULTILINE):
block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
vcard = block.replace("\n", "\r\n") + "\r\n"
url = base_url + "/" + quote(uid, safe="") + ".vcf"
try:
r = httpx.put(
url, data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth, timeout=15,
)
if r.status_code in (200, 201, 204):
imported += 1
else:
failed += 1
logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
except Exception as e:
failed += 1
logger.error(f"Import PUT {uid} failed: {e}")
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": len(blocks)}
def _import_csv_contacts(text: str) -> Dict:
"""Import contacts from CSV. Supports common headers:
name/full_name/display_name, email/email_address/e-mail, phone/tel.
Falls back to first columns as name,email,phone when no headers exist."""
raw = (text or "").strip()
if not raw:
return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
try:
sample = raw[:2048]
dialect = csv.Sniffer().sniff(sample)
except Exception:
dialect = csv.excel
stream = io.StringIO(raw)
try:
has_header = csv.Sniffer().has_header(raw[:2048])
except Exception:
has_header = True
rows = []
if has_header:
reader = csv.DictReader(stream, dialect=dialect)
for row in reader:
lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
name = (
lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
or lowered.get("display name") or lowered.get("display_name")
or lowered.get("fn") or ""
)
email = (
lowered.get("email") or lowered.get("email address")
or lowered.get("email_address") or lowered.get("e-mail")
or lowered.get("mail") or ""
)
phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
rows.append((name, email, phone))
else:
stream.seek(0)
reader = csv.reader(stream, dialect=dialect)
for row in reader:
cols = [(c or "").strip() for c in row]
if not any(cols):
continue
rows.append((
cols[0] if len(cols) > 0 else "",
cols[1] if len(cols) > 1 else "",
cols[2] if len(cols) > 2 else "",
))
imported = 0
failed = 0
total = 0
existing_emails = {
e.lower()
for c in _fetch_contacts()
for e in (c.get("emails") or [])
if e
}
for name, email, phone in rows:
email = (email or "").strip()
name = (name or "").strip() or (email.split("@")[0] if email else "")
if not email:
continue
total += 1
if email.lower() in existing_emails:
continue
ok = _create_contact(name, email)
if ok:
imported += 1
existing_emails.add(email.lower())
# If the CSV had a phone number, rewrite the just-created row
# through the richer update path so phone lands in CardDAV too.
if phone:
try:
contacts = _fetch_contacts(force=True)
created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
if created and created.get("uid"):
_update_contact(created["uid"], name, [email], [phone])
except Exception:
pass
else:
failed += 1
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": total}
def _contacts_to_vcf(contacts: List[Dict]) -> str:
return "".join(
_build_vcard(
c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
"",
uid=c.get("uid") or str(uuid.uuid4()),
emails=c.get("emails") or [],
phones=c.get("phones") or [],
)
for c in contacts
)
def _contacts_to_csv(contacts: List[Dict]) -> str:
out = io.StringIO()
writer = csv.writer(out)
writer.writerow(["name", "email", "phone"])
for c in contacts:
emails = c.get("emails") or [""]
phones = c.get("phones") or [""]
max_len = max(len(emails), len(phones), 1)
for i in range(max_len):
writer.writerow([
c.get("name") or "",
emails[i] if i < len(emails) else "",
phones[i] if i < len(phones) else "",
])
return out.getvalue()
def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
"""Rewrite an existing contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
found = False
out = []
for c in contacts:
if c.get("uid") == uid:
# Preserve existing address when caller passes "" (only
# updating name/emails/phones, not touching address).
addr = address if address else c.get("address", "")
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
found = True
else:
out.append(c)
if not found:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
_save_local_contacts(out)
return True
vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
# Use the real resource href (handles externally-created contacts whose
# filename != UID); falls back to the <uid>.vcf guess.
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to update contact: {e}")
return False
def _delete_contact(uid: str) -> bool:
"""Delete a contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
remaining = [c for c in contacts if c.get("uid") != uid]
_save_local_contacts(remaining)
return True
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.delete(url, auth=auth, timeout=10)
if r.status_code in (200, 204, 404):
# Invalidate cache so the next fetch sees the server truth.
_contact_cache["fetched_at"] = None
# Verify: force a fresh fetch and check the UID is actually gone.
# A 404 on the guessed URL ({uid}.vcf) can mean the contact
# lives at a different resource URL — the DELETE missed it but
# we'd silently report success. This check catches that.
fresh = _fetch_contacts(force=True)
still_there = any(c.get("uid") == uid for c in fresh)
if still_there:
logger.warning(
f"CardDAV DELETE reported success for {uid} "
f"but UID still present after re-fetch — "
f"resource URL may differ from {redact_url(url)}"
)
return False
if r.status_code == 404:
logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
return True
logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to delete contact: {e}")
return False
# ── Routes ──
def setup_contacts_routes():
router = APIRouter(prefix="/api/contacts", tags=["contacts"])
@router.get("/list")
async def list_contacts(_admin: str = Depends(require_admin)):
"""List all contacts."""
contacts = _fetch_contacts()
return {"contacts": contacts, "count": len(contacts)}
@router.get("/search")
async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
"""Search contacts by name or email. Returns up to 10 matches."""
contacts = _fetch_contacts()
if not q:
return {"results": []}
q_lower = q.lower()
results = []
for c in contacts:
if q_lower in c["name"].lower():
results.append(c)
continue
for em in c["emails"]:
if q_lower in em.lower():
results.append(c)
break
return {"results": results[:10]}
@router.post("/add")
async def add_contact(data: dict, _admin: str = Depends(require_admin)):
"""Add a new contact."""
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
phone = (data.get("phone") or "").strip()
phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
if phone and phone not in phones:
phones.insert(0, phone)
address = (data.get("address") or "").strip()
if not name and email:
name = email.split("@")[0]
if not name and not email and not phones and not address:
return {"success": False, "error": "Name, email, phone, or address required"}
if not name:
name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
contacts = _fetch_contacts()
for c in contacts:
if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
return {"success": True, "message": "Already exists", "contact": c}
if phones and any(p in (c.get("phones") or []) for p in phones):
return {"success": True, "message": "Already exists", "contact": c}
create_params = inspect.signature(_create_contact).parameters
if "phones" in create_params:
ok = _create_contact(name, email, address, phones=phones)
elif len(create_params) >= 3:
ok = _create_contact(name, email, address)
else:
ok = _create_contact(name, email)
# If a phone was provided, do an immediate update to thread it
# through (the simple _create_contact signature only takes name +
# email + address; phones happen via update).
if ok and phones and "phones" not in create_params:
try:
fresh = _fetch_contacts(force=True)
created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
if created:
_update_contact(
created["uid"], name,
created.get("emails", []),
phones,
address,
)
except Exception:
pass
return {"success": ok}
@router.post("/import")
async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
"""Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
# Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
# in the JSON body) would otherwise reach .strip() and 500 with an
# AttributeError instead of degrading to a clean "no data" response.
text = str(data.get("vcf") or data.get("text") or "")
csv_text = str(data.get("csv") or "")
if text.strip():
if "BEGIN:VCARD" not in text.upper():
return {"success": False, "error": "No vCard data found"}
result = _import_vcards(text)
elif csv_text.strip():
result = _import_csv_contacts(csv_text)
else:
return {"success": False, "error": "No contact data found"}
result["success"] = result.get("imported", 0) > 0
return result
@router.get("/export")
async def export_contacts(
format: str = Query("vcf", pattern="^(vcf|csv)$"),
_admin: str = Depends(require_admin),
):
"""Export all contacts as vCard or CSV."""
contacts = _fetch_contacts(force=True)
if format == "csv":
content = _contacts_to_csv(contacts)
media_type = "text/csv; charset=utf-8"
filename = "odysseus-contacts.csv"
else:
content = _contacts_to_vcf(contacts)
media_type = "text/vcard; charset=utf-8"
filename = "odysseus-contacts.vcf"
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/config")
async def get_config(_admin: str = Depends(require_admin)):
cfg = _get_carddav_config()
# Mask password
if cfg["password"]:
cfg["password"] = "***"
return cfg
@router.put("/config")
async def update_config(data: dict, _admin: str = Depends(require_admin)):
settings = _load_settings()
for key in ("carddav_url", "carddav_username", "carddav_password"):
if key in data:
if key == "carddav_url" and str(data[key] or "").strip():
try:
settings[key] = _validate_carddav_url(data[key])
except ValueError as e:
raise HTTPException(400, str(e))
else:
value = data[key]
if key == "carddav_password" and value:
from src.secret_storage import encrypt
value = encrypt(value)
settings[key] = value
_save_settings(settings)
# Force re-fetch
_contact_cache["fetched_at"] = None
return {"success": True}
@router.delete("/clear")
async def clear_contacts(_admin: str = Depends(require_admin)):
"""Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
_save_local_contacts([])
return {"success": True}
# NOTE: the /{uid} routes are declared LAST so the literal paths above
# (/list, /search, /add, /config) win — otherwise PUT /config would
# match PUT /{uid} with uid="config".
@router.put("/{uid}")
async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
"""Edit an existing contact — name / emails / phones / address."""
name = (data.get("name") or "").strip()
emails = data.get("emails")
phones = data.get("phones")
if emails is None and data.get("email"):
emails = [data["email"]]
emails = [e.strip() for e in (emails or []) if e and e.strip()]
phones = [p.strip() for p in (phones or []) if p and p.strip()]
address = (data.get("address") or "").strip()
if not name and not emails and not address:
return {"success": False, "error": "Name, email, or address required"}
if not name and emails:
name = emails[0].split("@")[0]
ok = _update_contact(uid, name, emails, phones, address)
return {"success": ok}
@router.delete("/{uid}")
async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
"""Delete a contact by UID."""
if not uid:
return {"success": False, "error": "UID required"}
ok = _delete_contact(uid)
return {"success": ok}
return router
routes/contacts_routes.py
+877 -8
View File
@@ -1,13 +1,882 @@
"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py.
"""
contacts_routes.py
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``,
``importlib.import_module("routes.contacts_routes")``, and string-targeted
monkeypatches all operate on the same object the application actually uses.
CardDAV contacts integration. Reads from local Radicale, supports
search and adding new contacts.
"""
import sys as _sys
import re
import logging
import uuid
import json
import csv
import io
import os
import inspect
import httpx
from pathlib import Path
from datetime import datetime
from urllib.parse import urljoin, urlparse, urlunparse
from routes.contacts import contacts_routes as _canonical # noqa: F401
from fastapi import APIRouter, Query, Depends, Response, HTTPException
from typing import List, Dict, Optional
_sys.modules[__name__] = _canonical
from core.middleware import require_admin
from src.url_safety import check_outbound_url
logger = logging.getLogger(__name__)
from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
DATA_DIR = Path(_DATA_DIR)
SETTINGS_FILE = Path(_SETTINGS_FILE)
LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
def _load_settings():
if SETTINGS_FILE.exists():
return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
return {}
def _save_settings(settings):
from core.atomic_io import atomic_write_json
atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
def _get_carddav_config():
import os
settings = _load_settings()
password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
if password and "carddav_password" in settings:
from src.secret_storage import decrypt
password = decrypt(password)
return {
"url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
"username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
"password": password,
}
def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
cfg = cfg or _get_carddav_config()
return bool((cfg.get("url") or "").strip())
def _validate_carddav_url(url: str) -> str:
cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
ok, reason = check_outbound_url(
cleaned,
block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
)
if not ok:
raise ValueError(f"Rejected CardDAV URL: {reason}")
return cleaned
def _carddav_base_url(cfg: Dict) -> str:
return _validate_carddav_url(cfg.get("url") or "")
def _normalize_contact(contact: Dict) -> Dict:
emails = []
for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
e = str(e or "").strip()
if e and e not in emails:
emails.append(e)
phones = []
for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
p = str(p or "").strip()
if p and p not in phones:
phones.append(p)
name = str(contact.get("name") or "").strip()
if not name and emails:
name = emails[0].split("@")[0]
address = str(contact.get("address") or "").strip()
return {
"uid": str(contact.get("uid") or uuid.uuid4()),
"name": name,
"emails": emails,
"phones": phones,
"address": address,
}
def _load_local_contacts() -> List[Dict]:
try:
if not LOCAL_CONTACTS_FILE.exists():
return []
data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
rows = data.get("contacts", data) if isinstance(data, dict) else data
return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
except Exception as e:
logger.error(f"Failed to load local contacts: {e}")
return []
def _save_local_contacts(contacts: List[Dict]) -> None:
from core.atomic_io import atomic_write_json
DATA_DIR.mkdir(parents=True, exist_ok=True)
atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
_contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
_contact_cache["fetched_at"] = datetime.utcnow()
# ── vCard parsing ──
def _vunesc(value: str) -> str:
"""Reverse _vesc() — turn escaped vCard text back into the raw value.
Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
if not value:
return value
out = []
i = 0
while i < len(value):
ch = value[i]
if ch == "\\" and i + 1 < len(value):
nxt = value[i + 1]
if nxt in ("n", "N"):
out.append("\n")
elif nxt in (",", ";", "\\"):
out.append(nxt)
else:
out.append(nxt)
i += 2
else:
out.append(ch)
i += 1
return "".join(out)
def _parse_vcards(text: str) -> List[Dict]:
"""Parse a stream of vCards into dicts with name, email, phone."""
contacts = []
for block in re.split(r"BEGIN:VCARD", text):
if not block.strip():
continue
contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
for line in block.split("\n"):
line = line.strip()
# Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
# that Apple Contacts / iCloud / many CardDAV servers emit by
# default — without this the property-name checks below miss those
# lines and silently drop the email / phone. The group token only
# precedes the property name, so it is safe to strip for matching
# and value extraction, and a no-op for non-grouped lines.
name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
if name_part.startswith("FN:") or name_part.startswith("FN;"):
contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
elif name_part.startswith("EMAIL"):
# Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
if ":" in name_part:
email_addr = _vunesc(name_part.split(":", 1)[1])
if email_addr and email_addr not in contact["emails"]:
contact["emails"].append(email_addr)
elif name_part.startswith("TEL"):
if ":" in name_part:
phone = _vunesc(name_part.split(":", 1)[1])
if phone and phone not in contact["phones"]:
contact["phones"].append(phone)
elif name_part.startswith("ADR"):
# vCard ADR is 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
# Recover a human-readable string by joining non-empty
# components with ", ".
if ":" in name_part:
raw = name_part.split(":", 1)[1]
parts = [_vunesc(p).strip() for p in raw.split(";")]
contact["address"] = ", ".join(p for p in parts if p)
elif name_part.startswith("UID:"):
contact["uid"] = _vunesc(name_part[4:])
if contact["name"] or contact["emails"]:
contacts.append(contact)
return contacts
def _vesc(value: str) -> str:
"""Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
or any value containing a newline produces a malformed vCard (broken
N/FN fields) or could inject arbitrary properties."""
return (
(value or "")
.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "")
.replace(",", "\\,")
.replace(";", "\\;")
)
def _build_vcard(name: str, email: str, uid: Optional[str] = None,
emails: Optional[List[str]] = None,
phones: Optional[List[str]] = None,
address: Optional[str] = None) -> str:
"""Build a vCard. Accepts either a single `email` (legacy callers) or
full `emails`/`phones` lists (edit path). The first email is marked
PREF=1. All values are RFC-6350-escaped."""
if not uid:
uid = str(uuid.uuid4())
# Normalize email lists — `email` arg is a convenience for single-email
# creation; `emails` (if given) is authoritative.
email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
# Try to split name into first/last
parts = name.strip().split()
if len(parts) >= 2:
first = parts[0]
last = " ".join(parts[1:])
else:
first = name
last = ""
# N field is structured (5 components separated by ';') — escape each
# component individually so a comma in the name doesn't split it.
n_field = f"{_vesc(last)};{_vesc(first)};;;"
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"UID:{_vesc(uid)}",
f"FN:{_vesc(name)}",
f"N:{n_field}",
]
for i, em in enumerate(email_list):
# First email is the preferred one.
lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
for ph in phone_list:
lines.append(f"TEL:{_vesc(ph)}")
# Address: stuff the whole human-readable string into the street
# component of ADR. vCard ADR has 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
addr = (address or "").strip()
if addr:
lines.append(f"ADR:;;{_vesc(addr)};;;;")
lines.append("END:VCARD")
return "\r\n".join(lines) + "\r\n"
# ── In-memory cache ──
_contact_cache = {"contacts": [], "fetched_at": None}
def _abs_url(href: str) -> str:
"""Combine a multistatus <href> (an absolute path like
/user/contacts/x.vcf) with the configured CardDAV server origin so we
get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
for the configured origin; a cross-origin href is treated as a path on the
configured server so a malicious CardDAV response cannot redirect later
writes/deletes to cloud metadata or another host."""
cfg = _get_carddav_config()
base = _carddav_base_url(cfg)
base_p = urlparse(base)
joined = urljoin(base.rstrip("/") + "/", href or "")
joined_p = urlparse(joined)
if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
return _validate_carddav_url(joined)
# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
# alongside the resource href. Lets us map each contact's UID to the real
# server resource path (which is NOT always <uid>.vcf for contacts created
# by other clients).
_ADDRESSBOOK_QUERY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
'<D:prop><D:getetag/><C:address-data/></D:prop>'
'<C:filter/>'
'</C:addressbook-query>'
)
def _fetch_via_report(cfg, auth):
"""Try a CardDAV REPORT addressbook-query — returns contacts WITH an
`href` field, or None if the server doesn't support it / errors."""
from defusedxml import ElementTree as ET
try:
r = httpx.request(
"REPORT", cfg["url"],
content=_ADDRESSBOOK_QUERY.encode("utf-8"),
headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
auth=auth, timeout=10,
)
if r.status_code not in (207, 200):
return None
root = ET.fromstring(r.text)
ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
out = []
for resp in root.findall("D:response", ns):
href_el = resp.find("D:href", ns)
data_el = resp.find(".//C:address-data", ns)
if href_el is None or data_el is None or not (data_el.text or "").strip():
continue
parsed = _parse_vcards(data_el.text)
if not parsed:
continue
c = parsed[0]
c["href"] = href_el.text.strip()
out.append(c)
# If the REPORT parsed to ZERO contacts, don't trust it — some
# CardDAV servers treat an empty <filter/> as "match nothing" and
# return a valid-but-empty 207. Return None so the caller falls
# back to the plain GET (which lists everything). A genuinely empty
# address book just costs one extra GET that also returns nothing.
if not out:
return None
return out
except Exception as e:
logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
return None
def _fetch_contacts(force=False):
"""Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
if not force and _contact_cache["fetched_at"]:
age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
if age < 60:
return _contact_cache["contacts"]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
try:
cfg["url"] = _carddav_base_url(cfg)
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
# Preferred path: REPORT gives us hrefs for reliable edit/delete.
contacts = _fetch_via_report(cfg, auth)
if contacts is None:
# Fallback: plain GET, concatenated vCards, no hrefs.
r = httpx.get(cfg["url"], auth=auth, timeout=10)
if r.status_code != 200:
logger.warning(f"CardDAV returned {r.status_code}")
return _contact_cache["contacts"]
contacts = _parse_vcards(r.text)
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
except Exception as e:
logger.error(f"Failed to fetch contacts: {e}")
return _contact_cache["contacts"]
def _resolve_resource_url(uid: str) -> str:
"""Map a contact UID to its real CardDAV resource URL. Uses the href
captured during fetch when available (handles contacts whose filename
!= UID); falls back to the <uid>.vcf guess for app-created contacts or
when no href is known."""
def _lookup():
for c in _contact_cache.get("contacts", []):
if c.get("uid") == uid and c.get("href"):
return _abs_url(c["href"])
return None
found = _lookup()
if found:
return found
# Not in cache (or no href) — refresh once and retry before guessing.
try:
_fetch_contacts(force=True)
except Exception:
pass
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str, address: str = "") -> bool:
"""Add a new contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = (email or "").strip().lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid, address=address)
try:
url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
# Invalidate cache
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to create contact: {e}")
return False
def _vcard_url(uid: str) -> str:
"""The CardDAV resource URL for a given contact UID. The uid is URL-
encoded so a value containing '/', '..' or other path chars can't
escape the collection and target an arbitrary CardDAV resource."""
from urllib.parse import quote
cfg = _get_carddav_config()
return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
def _import_vcards(text: str) -> Dict:
"""Import a (possibly multi-card) .vcf blob. Each card is PUT to the
CardDAV server PRESERVING its full original content (ADR/ORG/photo/
etc.) — we don't rebuild it, just ensure it has VERSION + UID and
normalize line endings. Returns {imported, failed, total}."""
from urllib.parse import quote
cfg = _get_carddav_config()
if not cfg.get("url"):
parsed = _parse_vcards(text)
contacts = _load_local_contacts()
existing = {
e.lower()
for c in contacts
for e in (c.get("emails") or [])
if e
}
imported = 0
for c in parsed:
emails = [e for e in (c.get("emails") or []) if e]
if emails and any(e.lower() in existing for e in emails):
continue
contacts.append(_normalize_contact(c))
for e in emails:
existing.add(e.lower())
imported += 1
if imported:
_save_local_contacts(contacts)
return {"imported": imported, "failed": 0, "total": len(parsed)}
try:
base_url = _carddav_base_url(cfg)
except ValueError as e:
logger.warning("CardDAV import URL rejected: %s", e)
return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
# Split into individual cards. re.split drops the BEGIN line, so we
# re-add it. Normalize CRLF.
raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
blocks = []
for chunk in raw.split("BEGIN:VCARD"):
chunk = chunk.strip()
if not chunk:
continue
# Trim anything after END:VCARD (defensive).
end = chunk.upper().find("END:VCARD")
body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
blocks.append("BEGIN:VCARD\n" + body)
imported = 0
failed = 0
for block in blocks:
# Extract or assign a UID.
m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
if not m:
# Inject a UID right after the VERSION line (or after BEGIN).
if re.search(r"^VERSION:", block, re.MULTILINE):
block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
else:
block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
elif not re.search(r"^VERSION:", block, re.MULTILINE):
block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
vcard = block.replace("\n", "\r\n") + "\r\n"
url = base_url + "/" + quote(uid, safe="") + ".vcf"
try:
r = httpx.put(
url, data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth, timeout=15,
)
if r.status_code in (200, 201, 204):
imported += 1
else:
failed += 1
logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
except Exception as e:
failed += 1
logger.error(f"Import PUT {uid} failed: {e}")
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": len(blocks)}
def _import_csv_contacts(text: str) -> Dict:
"""Import contacts from CSV. Supports common headers:
name/full_name/display_name, email/email_address/e-mail, phone/tel.
Falls back to first columns as name,email,phone when no headers exist."""
raw = (text or "").strip()
if not raw:
return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
try:
sample = raw[:2048]
dialect = csv.Sniffer().sniff(sample)
except Exception:
dialect = csv.excel
stream = io.StringIO(raw)
try:
has_header = csv.Sniffer().has_header(raw[:2048])
except Exception:
has_header = True
rows = []
if has_header:
reader = csv.DictReader(stream, dialect=dialect)
for row in reader:
lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
name = (
lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
or lowered.get("display name") or lowered.get("display_name")
or lowered.get("fn") or ""
)
email = (
lowered.get("email") or lowered.get("email address")
or lowered.get("email_address") or lowered.get("e-mail")
or lowered.get("mail") or ""
)
phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
rows.append((name, email, phone))
else:
stream.seek(0)
reader = csv.reader(stream, dialect=dialect)
for row in reader:
cols = [(c or "").strip() for c in row]
if not any(cols):
continue
rows.append((
cols[0] if len(cols) > 0 else "",
cols[1] if len(cols) > 1 else "",
cols[2] if len(cols) > 2 else "",
))
imported = 0
failed = 0
total = 0
existing_emails = {
e.lower()
for c in _fetch_contacts()
for e in (c.get("emails") or [])
if e
}
for name, email, phone in rows:
email = (email or "").strip()
name = (name or "").strip() or (email.split("@")[0] if email else "")
if not email:
continue
total += 1
if email.lower() in existing_emails:
continue
ok = _create_contact(name, email)
if ok:
imported += 1
existing_emails.add(email.lower())
# If the CSV had a phone number, rewrite the just-created row
# through the richer update path so phone lands in CardDAV too.
if phone:
try:
contacts = _fetch_contacts(force=True)
created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
if created and created.get("uid"):
_update_contact(created["uid"], name, [email], [phone])
except Exception:
pass
else:
failed += 1
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": total}
def _contacts_to_vcf(contacts: List[Dict]) -> str:
return "".join(
_build_vcard(
c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
"",
uid=c.get("uid") or str(uuid.uuid4()),
emails=c.get("emails") or [],
phones=c.get("phones") or [],
)
for c in contacts
)
def _contacts_to_csv(contacts: List[Dict]) -> str:
out = io.StringIO()
writer = csv.writer(out)
writer.writerow(["name", "email", "phone"])
for c in contacts:
emails = c.get("emails") or [""]
phones = c.get("phones") or [""]
max_len = max(len(emails), len(phones), 1)
for i in range(max_len):
writer.writerow([
c.get("name") or "",
emails[i] if i < len(emails) else "",
phones[i] if i < len(phones) else "",
])
return out.getvalue()
def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
"""Rewrite an existing contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
found = False
out = []
for c in contacts:
if c.get("uid") == uid:
# Preserve existing address when caller passes "" (only
# updating name/emails/phones, not touching address).
addr = address if address else c.get("address", "")
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
found = True
else:
out.append(c)
if not found:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
_save_local_contacts(out)
return True
vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
# Use the real resource href (handles externally-created contacts whose
# filename != UID); falls back to the <uid>.vcf guess.
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to update contact: {e}")
return False
def _delete_contact(uid: str) -> bool:
"""Delete a contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
remaining = [c for c in contacts if c.get("uid") != uid]
_save_local_contacts(remaining)
return True
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.delete(url, auth=auth, timeout=10)
if r.status_code in (200, 204):
_contact_cache["fetched_at"] = None
return True
if r.status_code == 404:
# Resource not found at the resolved URL. With href resolution
# this should be rare (genuinely already deleted). Invalidate
# the cache and report success so the UI doesn't keep a ghost.
logger.info(f"CardDAV DELETE 404 for {uid} — treating as already gone")
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to delete contact: {e}")
return False
# ── Routes ──
def setup_contacts_routes():
router = APIRouter(prefix="/api/contacts", tags=["contacts"])
@router.get("/list")
async def list_contacts(_admin: str = Depends(require_admin)):
"""List all contacts."""
contacts = _fetch_contacts()
return {"contacts": contacts, "count": len(contacts)}
@router.get("/search")
async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
"""Search contacts by name or email. Returns up to 10 matches."""
contacts = _fetch_contacts()
if not q:
return {"results": []}
q_lower = q.lower()
results = []
for c in contacts:
if q_lower in c["name"].lower():
results.append(c)
continue
for em in c["emails"]:
if q_lower in em.lower():
results.append(c)
break
return {"results": results[:10]}
@router.post("/add")
async def add_contact(data: dict, _admin: str = Depends(require_admin)):
"""Add a new contact."""
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
phone = (data.get("phone") or "").strip()
address = (data.get("address") or "").strip()
if not email:
return {"success": False, "error": "Email required"}
# Check if already exists by email
if email:
contacts = _fetch_contacts()
for c in contacts:
if email.lower() in [e.lower() for e in c["emails"]]:
return {"success": True, "message": "Already exists", "contact": c}
if not name:
name = email.split("@")[0]
create_params = inspect.signature(_create_contact).parameters
if len(create_params) >= 3:
ok = _create_contact(name, email, address)
else:
ok = _create_contact(name, email)
# If a phone was provided, do an immediate update to thread it
# through (the simple _create_contact signature only takes name +
# email + address; phones happen via update).
if ok and phone:
try:
fresh = _fetch_contacts(force=True)
created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
if created:
_update_contact(
created["uid"], name,
created.get("emails", []),
[phone],
address,
)
except Exception:
pass
return {"success": ok}
@router.post("/import")
async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
"""Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
# Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
# in the JSON body) would otherwise reach .strip() and 500 with an
# AttributeError instead of degrading to a clean "no data" response.
text = str(data.get("vcf") or data.get("text") or "")
csv_text = str(data.get("csv") or "")
if text.strip():
if "BEGIN:VCARD" not in text.upper():
return {"success": False, "error": "No vCard data found"}
result = _import_vcards(text)
elif csv_text.strip():
result = _import_csv_contacts(csv_text)
else:
return {"success": False, "error": "No contact data found"}
result["success"] = result.get("imported", 0) > 0
return result
@router.get("/export")
async def export_contacts(
format: str = Query("vcf", pattern="^(vcf|csv)$"),
_admin: str = Depends(require_admin),
):
"""Export all contacts as vCard or CSV."""
contacts = _fetch_contacts(force=True)
if format == "csv":
content = _contacts_to_csv(contacts)
media_type = "text/csv; charset=utf-8"
filename = "odysseus-contacts.csv"
else:
content = _contacts_to_vcf(contacts)
media_type = "text/vcard; charset=utf-8"
filename = "odysseus-contacts.vcf"
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/config")
async def get_config(_admin: str = Depends(require_admin)):
cfg = _get_carddav_config()
# Mask password
if cfg["password"]:
cfg["password"] = "***"
return cfg
@router.put("/config")
async def update_config(data: dict, _admin: str = Depends(require_admin)):
settings = _load_settings()
for key in ("carddav_url", "carddav_username", "carddav_password"):
if key in data:
if key == "carddav_url" and str(data[key] or "").strip():
try:
settings[key] = _validate_carddav_url(data[key])
except ValueError as e:
raise HTTPException(400, str(e))
else:
value = data[key]
if key == "carddav_password" and value:
from src.secret_storage import encrypt
value = encrypt(value)
settings[key] = value
_save_settings(settings)
# Force re-fetch
_contact_cache["fetched_at"] = None
return {"success": True}
@router.delete("/clear")
async def clear_contacts(_admin: str = Depends(require_admin)):
"""Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
_save_local_contacts([])
return {"success": True}
# NOTE: the /{uid} routes are declared LAST so the literal paths above
# (/list, /search, /add, /config) win — otherwise PUT /config would
# match PUT /{uid} with uid="config".
@router.put("/{uid}")
async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
"""Edit an existing contact — name / emails / phones / address."""
name = (data.get("name") or "").strip()
emails = data.get("emails")
phones = data.get("phones")
if emails is None and data.get("email"):
emails = [data["email"]]
emails = [e.strip() for e in (emails or []) if e and e.strip()]
phones = [p.strip() for p in (phones or []) if p and p.strip()]
address = (data.get("address") or "").strip()
if not name and not emails and not address:
return {"success": False, "error": "Name, email, or address required"}
if not name and emails:
name = emails[0].split("@")[0]
ok = _update_contact(uid, name, emails, phones, address)
return {"success": ok}
@router.delete("/{uid}")
async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
"""Delete a contact by UID."""
if not uid:
return {"success": False, "error": "UID required"}
ok = _delete_contact(uid)
return {"success": ok}
return router
routes/cookbook_helpers.py
+39 -273
View File
@@ -439,30 +439,15 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
" if f.is_file(): nf += 1; sz += f.stat().st_size",
" if f.name.endswith('.incomplete'): ic = True",
" snap = os.path.join(cache, d, 'snapshots')",
" def snapshot_size():",
" total, count, incomplete = 0, 0, False",
" seen_real = set()",
" # Windows HF cache stores files directly in snapshots/; blobs/ may be empty.",
" # Fallback: scan snapshots for real files when blobs yielded nothing.",
" if sz == 0 and os.path.isdir(snap):",
" for sd in os.listdir(snap):",
" sf = os.path.join(snap, sd)",
" if not os.path.isdir(sf): continue",
" for root, dirs, fns in safe_walk(sf):",
" for fn in fns:",
" fp = os.path.join(root, fn)",
" if fn.endswith('.incomplete'): incomplete = True",
" try:",
" real = os.path.realpath(fp)",
" if real in seen_real: continue",
" seen_real.add(real)",
" total += os.path.getsize(real)",
" count += 1",
" except Exception:",
" pass",
" return total, count, incomplete",
" # Some HF caches (macOS/MLX/Xet-style) keep blobs elsewhere or expose",
" # snapshot symlinks only. Size snapshots too when blob accounting is empty.",
" if sz == 0 and os.path.isdir(snap):",
" sz2, nf2, ic2 = snapshot_size()",
" sz, nf, ic = sz2, nf2, ic or ic2",
" for f in os.scandir(sf):",
" if f.is_file(): nf += 1; sz += f.stat().st_size",
" if f.name.endswith('.incomplete'): ic = True",
" is_diffusion = False; gguf_files = []",
" if os.path.isdir(snap):",
" for sd in os.listdir(snap):",
@@ -486,18 +471,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
" add('/app/.cache/huggingface/hub')",
f" add({add_hf_cache!r})" if add_hf_cache else "",
" return candidates",
"def normalize_model_dir(p):",
" p = os.path.expanduser((p or '').strip())",
" if not p: return p",
" if os.path.isdir(p) or os.path.isabs(p): return p",
" # Users often paste Linux absolute paths without the leading slash.",
" # Treat home/<user>/... as /home/<user>/... so remote scans work.",
" if p.startswith(('home/', 'mnt/', 'media/', 'data/', 'opt/', 'srv/', 'var/')):",
" prefixed = '/' + p",
" if os.path.isdir(prefixed): return prefixed",
" return p",
"def scan_dir(p):",
" p = normalize_model_dir(p)",
" if not os.path.isdir(p) or not safe_path(p): return",
" for d in sorted(os.listdir(p)):",
" if d.startswith('.'): continue",
@@ -531,8 +505,6 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
" if u.startswith('KB'): return int(n * 1024)",
" return int(n)",
"def scan_ollama():",
" if any(m.get('is_ollama') for m in models): return",
" if os.name == 'nt' and not os.environ.get('ODYSSEUS_ALLOW_OLLAMA_CLI_SCAN'): return",
" if not shutil.which('ollama'): return",
" try:",
" p = subprocess.run(['ollama', 'list'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=6)",
@@ -563,11 +535,11 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
" models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})",
" return",
"for _hf_cache in hf_cache_paths(): scan_hf(_hf_cache)",
"scan_ollama_api()",
"scan_ollama()",
"scan_ollama_api()",
]
for model_dir in model_dirs or []:
lines.append(f"scan_dir({model_dir!r})")
lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))")
lines.append("print(json.dumps(models))")
return "\n".join(lines) + "\n"
@@ -584,22 +556,10 @@ def _bash_squote(v: str) -> str:
return v.replace("'", "'\\''")
# Shown by generated runner scripts when the ollama binary is missing on the
# target host. Must stay free of backticks/$( ) and be emitted single-quoted:
# an earlier version wrapped the install one-liner in backticks inside a
# double-quoted echo, which bash executed as command substitution and ran the
# system-wide installer (including on remote SSH hosts) instead of printing
# the hint.
OLLAMA_MISSING_HINT = (
"ERROR: Ollama not found on this server. Install it from "
"https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh"
)
# Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
# Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
_SERVE_CMD_ALLOWLIST = {
"vllm", "llama-server", "llama-server.exe", "llama_server", "llama.cpp", "ollama",
"vllm", "llama-server", "llama_server", "llama.cpp", "ollama",
"python", "python3",
"sglang", "lmdeploy",
"node", "npx",
@@ -615,16 +575,6 @@ _SERVE_CMD_ALLOWLIST = {
_GGUF_PRELUDE_RE = re.compile(
r'^MODEL_FILE=\$\([^\n]*?\)\s*&&\s*\{[^{}]*\}\s*\|\|\s*\{[^{}]*\}\s*&&\s*'
)
_SAFE_SUBSHELL_TEXT = r"[^'\n;&|`$()<>]+"
_SAFE_SUBSHELL_DQ_HOME_PATH = r'"\$HOME/[^"\n;&|`()<>]*"'
_SAFE_PRINTF_SUBSHELL_RE = re.compile(
rf"^\$\(printf[ \t]+%s[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|\$\{{HOME\}}'/{_SAFE_SUBSHELL_TEXT}')\)$"
)
_SAFE_FIND_MMPROJ_SUBSHELL_RE = re.compile(
rf"^\$\(find[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|{_SAFE_SUBSHELL_DQ_HOME_PATH}|{_SAFE_SUBSHELL_TEXT})"
r"[ \t]+-iname[ \t]+'mmproj\*\.gguf'"
r"(?:[ \t]+2>/dev/null)?[ \t]*\|[ \t]*sort[ \t]*\|[ \t]*head[ \t]+-1\)$"
)
_OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)")
_OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$")
_OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
@@ -725,13 +675,6 @@ def _check_serve_binary(seg: str) -> None:
)
def _is_safe_serve_subshell(subshell: str) -> bool:
return bool(
_SAFE_PRINTF_SUBSHELL_RE.fullmatch(subshell)
or _SAFE_FIND_MMPROJ_SUBSHELL_RE.fullmatch(subshell)
)
def _validate_serve_cmd(v: str | None) -> str | None:
"""Reject serve commands that aren't in the allowlist or contain shell metachars.
@@ -763,15 +706,15 @@ def _validate_serve_cmd(v: str | None) -> str | None:
_check_serve_binary(part.strip())
return v
# Otherwise: a single invocation — no shell metacharacters allowed. Replace
# only the exact command substitutions emitted by the Cookbook UI:
# $(printf %s 'safe-path') and the mmproj lookup
# $(find <path> -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1).
def _replace_safe_subshell(match: re.Match[str]) -> str:
subshell = match.group(0)
return "/placeholder/safe/path" if _is_safe_serve_subshell(subshell) else subshell
cleaned_v = re.sub(r"\$\([^()]*\)", _replace_safe_subshell, v)
# Otherwise: a single invocation — no shell metacharacters allowed.
# Temporarily replace safe $(printf %s ...) expressions with a placeholder
# to avoid triggering the metacharacter/command-injection checks.
cleaned_v = v
printf_matches = list(re.finditer(r"\$\(\s*printf\s+%s\s+([^\n()]*?)\)", v))
for match in printf_matches:
inner = match.group(1)
if not any(c in inner for c in (";", "&&", "||", "$(", "`")):
cleaned_v = cleaned_v.replace(match.group(0), "/placeholder/safe/path.gguf")
# (`$(` was the original intent; bare `$` is fine for shell-safe paths.)
if any(c in cleaned_v for c in (";", "&&", "||", "$(")):
@@ -841,149 +784,25 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
to hard-wire CUDA on Linux. That made ROCm hosts attempt a CUDA configure and
fail with "CUDA Toolkit not found" instead of building with HIP.
"""
# Try a prebuilt binary from llama.cpp's GitHub releases FIRST — no
# cmake/build-essential/git/CUDA-headers needed at all. The from-source
# build below stays as a fallback (custom flags, esoteric arch, no
# internet, etc). 30 seconds vs 5+ minutes of compile, and removes
# every OS-package dep from the launch path. Sets _odysseus_have_prebuilt=1
# on success; the existing build-tier if/elif chain below is gated on
# that variable so we never compile twice or shadow the prebuilt symlink.
runner_lines.append(' _odysseus_have_prebuilt=""')
runner_lines.append(' _odysseus_arch="$(uname -m)"')
runner_lines.append(' _odysseus_prebuilt_url=""')
runner_lines.append(' if command -v curl >/dev/null 2>&1 && [ "$_odysseus_arch" = "x86_64" ]; then')
runner_lines.append(' _odysseus_pat=""')
runner_lines.append(' _odysseus_has_nv_inline() { command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU "; }')
runner_lines.append(' _odysseus_has_vk_inline() { ldconfig -p 2>/dev/null | grep -q "libvulkan\\.so" || command -v vulkaninfo >/dev/null 2>&1 || [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ]; }')
runner_lines.append(' _odysseus_has_vkdev_inline() { ls /dev/dri/renderD* >/dev/null 2>&1 || (lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\'); }')
runner_lines.append(' if _odysseus_has_nv_inline; then')
runner_lines.append(' _odysseus_pat="ubuntu.*cuda"')
runner_lines.append(' elif _odysseus_has_vkdev_inline && _odysseus_has_vk_inline; then')
runner_lines.append(' _odysseus_pat="ubuntu.*vulkan"')
runner_lines.append(' else')
runner_lines.append(' _odysseus_pat="ubuntu-x64\\\\.zip"')
runner_lines.append(' fi')
runner_lines.append(' _odysseus_prebuilt_url="$(curl -fsSL --max-time 15 https://api.github.com/repos/ggml-org/llama.cpp/releases/latest 2>/dev/null | grep \'"browser_download_url"\' | cut -d\'"\' -f4 | grep -iE "$_odysseus_pat" | grep -iv "arm\\|aarch64" | head -1)"')
runner_lines.append(' fi')
# Accept any of unzip / bsdtar / python3 -m zipfile as the extractor.
# python3 is essentially always present on modern Linux, so this lets
# the prebuilt path work on minimal Ubuntu installs that lack `unzip`.
runner_lines.append(' if [ -n "$_odysseus_prebuilt_url" ] && (command -v unzip >/dev/null 2>&1 || command -v bsdtar >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1); then')
runner_lines.append(' echo "[odysseus] Found prebuilt llama-server: $_odysseus_prebuilt_url"')
runner_lines.append(' mkdir -p ~/bin "$HOME/.cache/odysseus/llama-cpp-prebuilt" && cd "$HOME/.cache/odysseus/llama-cpp-prebuilt"')
runner_lines.append(' rm -f llama-cpp.zip')
runner_lines.append(' if curl -fsSL --max-time 120 "$_odysseus_prebuilt_url" -o llama-cpp.zip && [ -s llama-cpp.zip ]; then')
runner_lines.append(' rm -rf build && mkdir -p build')
runner_lines.append(' if command -v unzip >/dev/null 2>&1; then unzip -qq -o llama-cpp.zip -d build; elif command -v bsdtar >/dev/null 2>&1; then bsdtar -xf llama-cpp.zip -C build; else python3 -c "import zipfile; zipfile.ZipFile(\\"llama-cpp.zip\\").extractall(\\"build\\")"; fi')
runner_lines.append(' _odysseus_extracted="$(find build -type f -name llama-server 2>/dev/null | head -1)"')
runner_lines.append(' if [ -n "$_odysseus_extracted" ]; then')
runner_lines.append(' chmod +x "$_odysseus_extracted"')
runner_lines.append(' ln -sf "$_odysseus_extracted" ~/bin/llama-server')
runner_lines.append(' _odysseus_libdir="$(dirname "$_odysseus_extracted")"')
runner_lines.append(' mkdir -p ~/.config && echo "export LD_LIBRARY_PATH=\\"$_odysseus_libdir:\\${LD_LIBRARY_PATH:-}\\"" > ~/.config/odysseus-llama-cpp-env')
runner_lines.append(' _odysseus_have_prebuilt=1')
runner_lines.append(' echo "[odysseus] Prebuilt llama-server installed at $_odysseus_extracted"')
runner_lines.append(' fi')
runner_lines.append(' fi')
runner_lines.append(' [ -z "$_odysseus_have_prebuilt" ] && echo "[odysseus] Prebuilt download/extract failed — falling back to from-source build."')
runner_lines.append(' elif [ -z "$_odysseus_prebuilt_url" ]; then')
runner_lines.append(' echo "[odysseus] No matching prebuilt llama-server for this host (arch=$_odysseus_arch) — will build from source."')
runner_lines.append(' fi')
runner_lines.append(' if [ -z "$_odysseus_have_prebuilt" ]; then')
# Detect pip-installed nvcc (from vLLM/nvidia CUDA wheels) and put it on PATH
# so cmake's CUDA configure can find it — BUT only when actual NVIDIA
# hardware is present. On AMD/Intel hosts the pip nvcc is a misleading
# leftover (no libcudart, no GPU it could target) and would otherwise
# send the build down the CUDA branch and fail with "CUDA Toolkit not
# found" instead of trying Vulkan.
runner_lines.append(' _odysseus_has_nvidia_hw() {')
runner_lines.append(' command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU " && return 0')
runner_lines.append(' ls /dev/nvidia* >/dev/null 2>&1 && return 0')
runner_lines.append(' lspci 2>/dev/null | grep -iE \'VGA|3D|Display\' | grep -iq nvidia && return 0')
runner_lines.append(' return 1')
runner_lines.append(' }')
runner_lines.append(' if _odysseus_has_nvidia_hw; then')
runner_lines.append(' for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do')
runner_lines.append(' [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break')
runner_lines.append(' done')
runner_lines.append(' fi')
# so cmake's CUDA configure can find it. We keep this after the ROCm/HIP
# check — a machine with both stacks should honor the native HIP toolchain on
# AMD hosts instead of accidentally preferring a stray nvcc wheel.
runner_lines.append(' for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do')
runner_lines.append(' [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break')
runner_lines.append(' done')
# rm -rf build so a prior poisoned CMakeCache.txt (e.g. from a failed CUDA
# or HIP attempt) doesn't cause the next configure to reuse stale settings.
runner_lines.append(' mkdir -p ~/bin')
# Try to install cmake / build-essential / git automatically before the
# build, but ONLY via passwordless sudo (`sudo -n`) — interactive sudo
# would hang a tmux-backgrounded serve task waiting for a password. If
# sudo asks for a password the install is skipped silently and the
# diagnosis pattern (cookbook_routes.py / cookbook_helpers.py) surfaces
# an explicit "install cmake" suggestion in the Cookbook diagnosis
# toolbar after the inevitable build failure.
runner_lines.append(' _odysseus_apt_bootstrap() {')
runner_lines.append(' local _missing=""')
runner_lines.append(' command -v cmake >/dev/null 2>&1 || _missing="$_missing cmake"')
runner_lines.append(' command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _missing="$_missing build-essential"')
runner_lines.append(' command -v git >/dev/null 2>&1 || _missing="$_missing git"')
runner_lines.append(' [ -z "$_missing" ] && return 0')
runner_lines.append(' if command -v apt-get >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via apt:$_missing"')
runner_lines.append(' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>&1 | tail -3')
runner_lines.append(' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $_missing 2>&1 | tail -5 || true')
runner_lines.append(' elif command -v pacman >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via pacman:$_missing"')
runner_lines.append(' local _pacpkgs="$(echo "$_missing" | sed -e \'s/build-essential/base-devel/g\')"')
runner_lines.append(' sudo -n pacman -Sy --needed --noconfirm $_pacpkgs 2>&1 | tail -5 || true')
runner_lines.append(' elif command -v dnf >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via dnf:$_missing"')
runner_lines.append(' local _dnfpkgs="$(echo "$_missing" | sed -e \'s/build-essential/gcc gcc-c++ make/g\')"')
runner_lines.append(' sudo -n dnf install -y $_dnfpkgs 2>&1 | tail -5 || true')
runner_lines.append(' else')
runner_lines.append(' echo "[odysseus] WARNING: missing build deps ($_missing) — passwordless sudo is unavailable, cannot auto-install. Cookbook Diagnosis will explain the fix after the build fails."')
runner_lines.append(' fi')
runner_lines.append(' }')
runner_lines.append(' _odysseus_apt_bootstrap')
runner_lines.append(' _odysseus_missing_build_deps=""')
runner_lines.append(' command -v cmake >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps cmake"')
runner_lines.append(' command -v git >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps git"')
runner_lines.append(' command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps build-essential"')
runner_lines.append(' if [ -n "$_odysseus_missing_build_deps" ]; then')
runner_lines.append(' echo "ERROR: llama.cpp source build needs missing packages:$_odysseus_missing_build_deps"')
runner_lines.append(' if command -v apt-get >/dev/null 2>&1; then')
runner_lines.append(' echo "Install on this host: sudo apt-get update && sudo apt-get install -y cmake build-essential git"')
runner_lines.append(' elif command -v pacman >/dev/null 2>&1; then')
runner_lines.append(' echo "Install on this host: sudo pacman -Sy --needed cmake base-devel git"')
runner_lines.append(' elif command -v dnf >/dev/null 2>&1; then')
runner_lines.append(' echo "Install on this host: sudo dnf install -y cmake gcc gcc-c++ make git"')
runner_lines.append(' fi')
runner_lines.append(' echo "Alternative: install a native llama-server on PATH, then relaunch."')
runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127')
runner_lines.append(' fi')
runner_lines.append(' cd ~/llama.cpp')
runner_lines.append(' _odysseus_has_vulkan() {')
runner_lines.append(' ldconfig -p 2>/dev/null | grep -q \'libvulkan\\.so\' && return 0')
runner_lines.append(' [ -e /usr/lib/libvulkan.so.1 ] && return 0')
runner_lines.append(' [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ] && return 0')
runner_lines.append(' command -v vulkaninfo >/dev/null 2>&1 && return 0')
runner_lines.append(' return 1')
runner_lines.append(' }')
runner_lines.append(' _odysseus_has_vulkan_device() {')
runner_lines.append(' ls /dev/dri/renderD* >/dev/null 2>&1 && return 0')
runner_lines.append(' lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\' && return 0')
runner_lines.append(' return 1')
runner_lines.append(' }')
# Backend preference: native ROCm/HIP > native CUDA > Vulkan > CPU.
# Vulkan is a portable fallback that works on AMD when ROCm isn't
# installed (e.g. Strix Halo) and on any vendor's discrete GPU, but
# it's ~30-40% slower than native HIP/CUDA for LLM inference — only
# pick it when no native toolchain is present.
runner_lines.append(' cd ~/llama.cpp && rm -rf build')
runner_lines.append(' if command -v hipconfig &>/dev/null || [ -d /opt/rocm ] || [ -n "$ROCM_PATH" ] || [ -n "$HIP_PATH" ]; then')
runner_lines.append(' rm -rf build')
runner_lines.append(' if command -v hipconfig &>/dev/null; then')
runner_lines.append(' export HIPCXX="${HIPCXX:-$(hipconfig -l)/clang}"')
runner_lines.append(' export HIP_PATH="${HIP_PATH:-$(hipconfig -R)}"')
runner_lines.append(' fi')
runner_lines.append(' echo "[odysseus] ROCm/HIP detected — building llama-server with HIP support..."')
runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_HIP=ON && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
runner_lines.append(' elif command -v nvcc &>/dev/null && _odysseus_has_nvidia_hw; then')
runner_lines.append(' rm -rf build')
runner_lines.append(' elif command -v nvcc &>/dev/null; then')
# nvcc alone is not sufficient — pip-installed CUDA wheels or incomplete
# tooling can expose nvcc without shipping libcudart, causing cmake to fail
# mid-build with "CUDA runtime library not found". Check cudart explicitly
@@ -1007,50 +826,31 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
runner_lines.append(' echo "[odysseus] Ensure libcudart is installed (e.g. cuda-runtime package) and visible via ldconfig or CUDA_HOME."')
runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
runner_lines.append(' fi')
runner_lines.append(' elif _odysseus_has_vulkan_device && _odysseus_has_vulkan; then')
runner_lines.append(' echo "[odysseus] Vulkan-capable GPU detected (no ROCm/CUDA toolchain installed) — building llama-server with Vulkan support..."')
runner_lines.append(' rm -rf build-vulkan')
runner_lines.append(' cmake -B build-vulkan -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON && cmake --build build-vulkan -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build-vulkan/bin/llama-server ~/bin/llama-server')
runner_lines.append(' else')
runner_lines.append(' echo "[odysseus] WARNING: no HIP/CUDA/Vulkan toolchain found — building llama-server for CPU only."')
runner_lines.append(' echo "[odysseus] WARNING: no HIP/CUDA toolchain found — building llama-server for CPU only."')
runner_lines.append(' echo "[odysseus] GPU inference will not be available for this llama.cpp build."')
runner_lines.append(' echo "[odysseus] Install Vulkan (libvulkan-dev) / ROCm for AMD GPUs or CUDA tooling for NVIDIA, then re-launch this serve task."')
runner_lines.append(' rm -rf build')
runner_lines.append(' echo "[odysseus] Install ROCm for AMD GPUs or vLLM/CUDA tooling for NVIDIA, then re-launch this serve task."')
runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
runner_lines.append(' fi')
runner_lines.append(' fi # end _odysseus_have_prebuilt guard')
def _llama_cpp_rebuild_cmd(update_source: bool = False) -> str:
def _llama_cpp_rebuild_cmd() -> str:
"""Shell command that clears the Cookbook-managed llama.cpp build.
Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build*``
Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build``
directory so the next llama.cpp serve recompiles from source, picking up a
CUDA or HIP toolchain if one is now available. The serve bootstrap only
builds when ``llama-server`` is missing from PATH, so without this an
existing CPU-only build is reused forever. When ``update_source`` is true,
the command also fast-forwards the Cookbook-managed ``~/llama.cpp`` checkout
if it exists. The rebuild itself happens on the next serve.
existing CPU-only build is reused forever. It deliberately installs and
downloads nothing; the rebuild itself happens on the next serve.
"""
update_cmd = ''
if update_source:
update_cmd = (
'if [ -d "$HOME/llama.cpp/.git" ]; then '
'git -C "$HOME/llama.cpp" pull --ff-only --depth 1 || '
'echo "[odysseus] WARNING: llama.cpp source update failed; clearing cached build anyway."; '
'elif command -v git >/dev/null 2>&1; then '
'git clone --depth 1 https://github.com/ggml-org/llama.cpp "$HOME/llama.cpp" || '
'echo "[odysseus] WARNING: llama.cpp clone failed; clearing cached build anyway."; '
'fi && '
)
return (
'mkdir -p "$HOME/bin" && '
f'{update_cmd}'
'rm -f "$HOME/bin/llama-server" && '
'rm -rf "$HOME/llama.cpp/build" "$HOME/llama.cpp/build-vulkan" && '
'rm -rf "$HOME/llama.cpp/build" && '
'echo "[odysseus] Cleared the cached llama.cpp build. '
'Re-launch the serve task to rebuild llama-server from source '
'(Vulkan, HIP, or CUDA will be used if a matching toolchain is now available)."'
'(CUDA or HIP will be used if a toolchain is now available)."'
)
@@ -1299,15 +1099,13 @@ def _diagnose_serve_output(text: str) -> dict | None:
[{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
),
(
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
r"Could not load any common_ops library|"
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
r"Please ensure sgl_kernel is properly installed",
"SGLang native kernel/runtime is missing or mismatched on this server.",
"SGLang native dependencies are missing on this server.",
[
{"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
{"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
{"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
{"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
],
),
(
@@ -1316,39 +1114,7 @@ def _diagnose_serve_output(text: str) -> dict | None:
[{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
),
(
r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
"MLX LM is not installed on this server.",
[{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
),
(
r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
"MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
[
{"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
{"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
],
),
# System build deps come BEFORE the generic llama.cpp catch-all so
# cmake / build-essential / git missing → a specific OS-package
# remediation instead of "install llama-cpp-python[server]" (which
# itself fails to compile when cmake is absent).
(
r"cmake: command not found|cmake.*not found.*[Cc]ould not",
"cmake is required to build llama.cpp from source but isn't installed on this server.",
[{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
(
r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler",
"A C/C++ compiler (build-essential) is required to build llama.cpp from source.",
[{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
(
r"^git: command not found",
"git is required to clone the llama.cpp source tree.",
[{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
(
r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'",
r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'|git: command not found|cmake: command not found",
"llama.cpp / llama-cpp-python dependencies are missing.",
[{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
routes/cookbook_routes.py
+182 -1404
View File
File diff suppressed because it is too large Load Diff
routes/document_helpers.py
+2 -8
View File
@@ -12,7 +12,6 @@ from pydantic import BaseModel
from core.database import Document, DocumentVersion
from core.database import Session as DbSession
from src.auth_helpers import _auth_disabled
from src.upload_handler import UploadHandler
logger = logging.getLogger(__name__)
@@ -29,7 +28,6 @@ class DocumentCreate(BaseModel):
class DocumentUpdate(BaseModel):
content: str
summary: Optional[str] = None
force_version: bool = False
class DocumentPatch(BaseModel):
title: Optional[str] = None
@@ -80,8 +78,6 @@ def _verify_doc_owner(db, doc: Document, user: str):
the session join for any not-yet-backfilled legacy row.
"""
if user is None:
if _auth_disabled():
return # Single-user / no-auth mode: allow access
raise HTTPException(403, "Authentication required")
if doc.owner is not None:
if doc.owner != user:
@@ -106,10 +102,8 @@ def _owner_session_filter(q, user):
The owner backfill runs in init_db before the app serves requests, so
by the time this filter is live there are no NULL-owner rows to leak;
we therefore match the owner strictly for authenticated callers."""
if not user:
if user == "" or _auth_disabled():
return q
we therefore match the owner strictly."""
if user is None:
return q.filter(False)
return q.filter(Document.owner == user)
routes/document_routes.py
+21 -114
View File
@@ -10,9 +10,8 @@ from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File,
from sqlalchemy import case, func, or_
from core.database import SessionLocal, Document, DocumentVersion
from core.database import Session as DbSession
from src.auth_helpers import get_current_user, _auth_disabled
from src.auth_helpers import get_current_user
from src.constants import MAIL_ATTACHMENTS_DIR
from src.upload_handler import reserve_upload_references
logger = logging.getLogger(__name__)
@@ -55,18 +54,6 @@ def _library_language_for_document(doc: Document) -> str:
return doc.language or "text"
def _email_source_key(content: str) -> tuple[str, str]:
"""Return the source email identity embedded in an email draft document."""
import re
text = content or ""
uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text)
folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text)
uid = (uid_m.group(1).strip() if uid_m else "")
folder = (folder_m.group(1).strip() if folder_m else "INBOX")
return uid, folder
from routes.document_helpers import (
DocumentCreate, DocumentUpdate, DocumentPatch,
_doc_to_dict, _version_to_dict,
@@ -79,14 +66,6 @@ from routes.document_helpers import (
def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
router = APIRouter(tags=["documents"])
def _reserve_document_uploads(user: Optional[str], content: str) -> None:
missing_id = reserve_upload_references(upload_handler, user, content)
if missing_id:
raise HTTPException(
409,
f"Referenced upload is no longer available: {missing_id}",
)
def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]):
if upload_handler is None:
return None
@@ -120,63 +99,24 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# the existing lenient path.
session = _get_session_or_404(db, req.session_id, user)
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
# If no language was supplied (e.g. cloning a doc whose language
# was never set), detect it from the content rather than storing
# NULL — which made the editor fall back to plain text. Defaults
# to markdown for prose.
language = req.language
if not language:
from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content
from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language
language = _sniff_doc_language(req.content)
else:
from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content
from src.agent_tools.document_tools import _looks_like_email_document
if _looks_like_email_document(req.content, req.title):
language = "email"
_reserve_document_uploads(user, req.content)
_assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
# Reply drafts are keyed to the source email. If a UI/tool path tries
# to create a second draft for the same email in the same chat,
# update the existing draft instead so quoted thread history stays
# attached to the visible document.
if language == "email" and req.session_id:
source_uid, source_folder = _email_source_key(req.content)
if source_uid:
candidates = (
db.query(Document)
.filter(Document.session_id == req.session_id)
.filter(Document.is_active == True)
.filter(Document.language == "email")
.order_by(Document.updated_at.desc())
.limit(25)
.all()
)
for existing in candidates:
old_uid, old_folder = _email_source_key(existing.current_content or "")
if old_uid != source_uid or old_folder != source_folder:
continue
merged = _coerce_email_document_content(existing.current_content or "", req.content)
if existing.current_content != merged:
new_ver = (existing.version_count or 1) + 1
existing.current_content = merged
existing.title = req.title or existing.title
existing.version_count = new_ver
db.add(DocumentVersion(
id=str(uuid.uuid4()),
document_id=existing.id,
version_number=new_ver,
content=merged,
summary="Updated existing email draft",
source="user",
))
db.commit()
db.refresh(existing)
return _doc_to_dict(existing)
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
doc = Document(
id=doc_id,
session_id=req.session_id,
@@ -448,8 +388,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
db = SessionLocal()
try:
if not user:
if not _auth_disabled():
raise HTTPException(403, "Authentication required")
raise HTTPException(403, "Authentication required")
# v2 review HIGH-9: raise 403 explicitly when the caller
# can't see this session, instead of returning [] which the
# UI treats identically to "no docs" and silently masks
@@ -564,8 +503,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
user = get_current_user(request)
try:
data = await request.json()
except Exception as e:
logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e)
except Exception:
data = {}
ids = data.get("ids") or []
if not ids:
@@ -630,24 +568,11 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
raise HTTPException(404, "Document not found")
_verify_doc_owner(db, doc, user)
incoming_content = req.content
from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document
is_email_doc = (
(doc.language or "").lower() == "email"
or _looks_like_email_document(doc.current_content or "", doc.title or "")
or _looks_like_email_document(req.content or "", doc.title or "")
)
if is_email_doc:
incoming_content = _coerce_email_document_content(doc.current_content or "", req.content)
doc.language = "email"
# Skip if content is identical unless the caller explicitly wants
# a checkpoint version from the current editor state.
if doc.current_content == incoming_content and not req.force_version:
# Skip if content is identical
if doc.current_content == req.content:
return _doc_to_dict(doc)
_reserve_document_uploads(user, incoming_content)
_assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler)
_assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
# Check if we can coalesce with the latest version
latest_ver = db.query(DocumentVersion).filter(
@@ -656,14 +581,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
now = datetime.now(timezone.utc)
coalesced = False
if latest_ver and latest_ver.source == "user" and not req.force_version:
if latest_ver and latest_ver.source == "user":
ver_time = latest_ver.created_at
if ver_time.tzinfo is None:
ver_time = ver_time.replace(tzinfo=timezone.utc)
age = (now - ver_time).total_seconds()
if age < VERSION_COALESCE_SECONDS:
# Update the existing version in-place
latest_ver.content = incoming_content
latest_ver.content = req.content
latest_ver.created_at = now
if req.summary:
latest_ver.summary = req.summary
@@ -675,14 +600,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
id=str(uuid.uuid4()),
document_id=doc_id,
version_number=new_ver,
content=incoming_content,
content=req.content,
summary=req.summary or "Manual edit",
source="user",
)
doc.version_count = new_ver
db.add(ver)
doc.current_content = incoming_content
doc.current_content = req.content
db.commit()
db.refresh(doc)
return _doc_to_dict(doc)
@@ -720,8 +645,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
try:
from src.agent_tools.document_tools import clear_active_document
clear_active_document(doc_id)
except Exception as e:
logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e)
except Exception:
pass
db.commit()
db.refresh(doc)
return _doc_to_dict(doc)
@@ -872,26 +797,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
from src.document_actions import _JUNK_TITLES
to_delete = []
now = datetime.now(timezone.utc)
for doc in docs:
created = doc.created_at
if created and created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
# Skip freshly created documents to avoid deleting them while the user is actively editing
if created and (now - created).total_seconds() < 900: # 15 minutes
continue
content = (doc.current_content or "").strip()
title_raw = (doc.title or "").strip()
title = title_raw.lower()
is_fresh_empty = (
not content
and created is not None
and (now - created).total_seconds() < 1800
)
if is_fresh_empty:
continue
# Strip markdown noise to get a "real" character count
stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE)
@@ -926,6 +835,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
to_delete.append(doc); deleted += 1; continue
if title in _JUNK_TITLES:
to_delete.append(doc); deleted += 1; continue
if real_len < 30:
to_delete.append(doc); deleted += 1; continue
if "\n" not in content and real_len < 50:
to_delete.append(doc); deleted += 1; continue
# Fix empty or placeholder titles on survivors
if not title_raw or title_raw == "Untitled":
@@ -1418,12 +1331,6 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
if not pdf_path:
raise HTTPException(404, f"Source PDF {upload_id} not found")
# Fail fast with a clear 503 if the optional PyMuPDF dependency
# is missing — fill_fields/stamp_annotations will otherwise
# raise RuntimeError deep inside and bubble out as a 500.
# Mirrors the convention in _load_pdf_viewer_fitz above.
_load_pdf_viewer_fitz()
values = parse_markdown_to_values(doc.current_content or "")
out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name
_to_unlink.append(out_path)
routes/email_helpers.py
+27 -228
View File
@@ -40,16 +40,6 @@ from src.secret_storage import decrypt as _decrypt
logger = logging.getLogger(__name__)
class EmailNotConfiguredError(RuntimeError):
"""Raised when an IMAP operation is attempted on an account that has no
inbox configured (e.g. a send-only / SMTP-only account).
Subclasses RuntimeError so existing broad ``except Exception`` handlers
keep working; callers that want to treat "no inbox" as an empty result
rather than a failure can catch this type specifically.
"""
def _xoauth2_raw(user: str, access_token: str) -> str:
"""The SASL XOAUTH2 initial-response string (unencoded).
@@ -235,9 +225,8 @@ def _strip_think(text: str) -> str:
"""
if not text:
return ""
from src.text_helpers import strip_think as _central, _THINK_TAG_RE
# Single linear tag check; the old closed/open `.search()` calls could ReDoS.
had_think = bool(_THINK_TAG_RE.search(text))
from src.text_helpers import strip_think as _central, _THINK_CLOSED_RE, _THINK_OPEN_RE, _THINK_TAG_RE
had_think = bool(_THINK_CLOSED_RE.search(text) or _THINK_OPEN_RE.search(text) or _THINK_TAG_RE.search(text))
return _central(text, prose=had_think, prompt_echo=True)
@@ -349,7 +338,7 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
row = db.query(_EA).filter(_EA.id == account_id).first()
if row is None:
raise HTTPException(404, "Account not found")
if not _account_visible_to_owner(row, owner):
if row.owner and row.owner != owner:
# Treat as 404 (not 403) so we don't leak existence.
raise HTTPException(404, "Account not found")
finally:
@@ -362,26 +351,6 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
logger.error(f"Account-owner check failed: {e}")
raise HTTPException(503, "Account check failed")
def _account_visible_to_owner(row, owner: str) -> bool:
"""Whether an authenticated `owner` may act on this EmailAccount row.
Mirrors the SQL predicate in `_get_email_config`'s
`_owner_or_matching_legacy_account`: a caller sees an account they own, or a
legacy owner-less account (owner NULL/"") only when its own mailbox
(`imap_user` / `from_address`) is the caller's. `email_accounts` is the one
owner-scoped table deliberately left out of the legacy-owner migration
backfill, so ownerless rows persist on multi-user deploys — making this the
gate that keeps one tenant off another's imported mailbox and its decrypted
IMAP/SMTP credentials."""
row_owner = getattr(row, "owner", None) or ""
if row_owner:
return row_owner == owner
return owner in {
getattr(row, "imap_user", None) or "",
getattr(row, "from_address", None) or "",
}
def _q(name: str) -> str:
"""Quote an IMAP mailbox name. Defensive: escapes `\\` and `"` and wraps
in double quotes so user-supplied folder names with spaces or quotes can't
@@ -444,19 +413,12 @@ SCHEDULED_DB = Path(SCHEDULED_EMAILS_DB)
OWNER_SCOPED_EMAIL_CACHE_TABLES = {
"email_summaries",
"email_ai_replies",
"email_translations",
"email_calendar_extractions",
"email_urgency_alerts",
"sender_signatures",
}
def email_translation_body_hash(body: str) -> str:
import hashlib as _hashlib
normalized = (body or "").strip()
return _hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest()
def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
owner = (owner or "").strip()
if owner:
@@ -464,34 +426,14 @@ def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
return "(owner = '' OR owner IS NULL)", ()
def _ensure_owner_scoped_email_cache_table(
conn,
table: str,
create_sql: str,
columns: list[str],
pk_columns: list[str] | None = None,
):
def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, columns: list[str]):
"""Rebuild legacy Message-ID-only cache tables with owner in the PK."""
desired_pk_cols = pk_columns or ["message_id", "owner"]
conn.execute(create_sql)
try:
info = conn.execute(f"PRAGMA table_info({table})").fetchall()
cols = [r[1] for r in info]
pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
for col in columns:
if col not in cols:
if col == "owner":
conn.execute(f"ALTER TABLE {table} ADD COLUMN owner TEXT DEFAULT ''")
elif col in {"event_uids"}:
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT '[]'")
elif col.startswith("has_") or col.endswith("_created") or col.endswith("_count"):
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} INTEGER DEFAULT 0")
elif col == "created_at":
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT ''")
else:
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT")
cols.append(col)
if "owner" in cols and pk_cols == desired_pk_cols:
if "owner" in cols and pk_cols == ["message_id", "owner"]:
return
conn.execute(f"ALTER TABLE {table} RENAME TO {table}__old")
@@ -624,25 +566,6 @@ def _init_scheduled_db():
PRIMARY KEY (message_id, owner)
)
""", ["message_id", "owner", "uid", "folder", "reply", "model_used", "created_at"])
_ensure_owner_scoped_email_cache_table(conn, "email_translations", """
CREATE TABLE IF NOT EXISTS email_translations (
body_hash TEXT,
owner TEXT DEFAULT '',
target_language TEXT DEFAULT 'English',
uid TEXT,
folder TEXT,
subject TEXT,
sender TEXT,
translation TEXT,
same_language INTEGER DEFAULT 0,
model_used TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (body_hash, owner, target_language)
)
""", [
"body_hash", "owner", "target_language", "uid", "folder", "subject", "sender",
"translation", "same_language", "model_used", "created_at",
], ["body_hash", "owner", "target_language"])
# Email tags / spam classification cache. SECURITY: keyed by
# (message_id, owner) because Message-IDs are GLOBAL (a newsletter goes
# to many users with the same Message-ID). Without owner-scoping, a
@@ -652,7 +575,6 @@ def _init_scheduled_db():
CREATE TABLE IF NOT EXISTS email_tags (
message_id TEXT,
owner TEXT DEFAULT '',
account_id TEXT DEFAULT '',
uid TEXT,
folder TEXT,
subject TEXT,
@@ -663,7 +585,7 @@ def _init_scheduled_db():
moved_to TEXT,
model_used TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (message_id, owner, account_id)
PRIMARY KEY (message_id, owner)
)
""")
# Backfill migration: older installs created the table with
@@ -671,35 +593,28 @@ def _init_scheduled_db():
# promote it into the PK by rebuild-copy-swap (SQLite can't ALTER PK).
try:
_cols = [r[1] for r in conn.execute("PRAGMA table_info(email_tags)")]
_pk_cols = [r[1] for r in sorted(conn.execute("PRAGMA table_info(email_tags)").fetchall(), key=lambda row: row[5] or 99) if r[5]]
if "owner" not in _cols:
# Add the column first so reads/writes don't break mid-migration.
conn.execute("ALTER TABLE email_tags ADD COLUMN owner TEXT DEFAULT ''")
_cols.append("owner")
if "account_id" not in _cols:
conn.execute("ALTER TABLE email_tags ADD COLUMN account_id TEXT DEFAULT ''")
_cols.append("account_id")
if _pk_cols != ["message_id", "owner", "account_id"]:
# Rebuild with account-aware composite PK. Existing rows get
# account_id='' and are still readable as legacy fallback rows;
# fresh task runs write exact account ids and no longer block each
# other when two accounts share a Message-ID.
# Rebuild with composite PK. Existing rows get owner='' (legacy
# single-user); the urgency scanner will overwrite as it
# re-classifies. No data loss.
conn.execute("""
CREATE TABLE IF NOT EXISTS email_tags__new (
message_id TEXT,
owner TEXT DEFAULT '',
account_id TEXT DEFAULT '',
uid TEXT, folder TEXT, subject TEXT, sender TEXT,
tags TEXT, spam_verdict INTEGER DEFAULT 0,
spam_reason TEXT, moved_to TEXT, model_used TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (message_id, owner, account_id)
PRIMARY KEY (message_id, owner)
)
""")
conn.execute("""
INSERT OR IGNORE INTO email_tags__new
(message_id, owner, account_id, uid, folder, subject, sender, tags,
(message_id, owner, uid, folder, subject, sender, tags,
spam_verdict, spam_reason, moved_to, model_used, created_at)
SELECT message_id, COALESCE(owner, ''), COALESCE(account_id, ''), uid, folder, subject,
SELECT message_id, COALESCE(owner, ''), uid, folder, subject,
sender, tags, spam_verdict, spam_reason, moved_to,
model_used, created_at
FROM email_tags
@@ -715,12 +630,11 @@ def _init_scheduled_db():
message_id TEXT,
owner TEXT DEFAULT '',
uid TEXT,
event_uids TEXT DEFAULT '[]',
events_created INTEGER DEFAULT 0,
created_at TEXT NOT NULL,
PRIMARY KEY (message_id, owner)
)
""", ["message_id", "owner", "uid", "event_uids", "events_created", "created_at"])
""", ["message_id", "owner", "uid", "events_created", "created_at"])
_ensure_owner_scoped_email_cache_table(conn, "email_urgency_alerts", """
CREATE TABLE IF NOT EXISTS email_urgency_alerts (
message_id TEXT,
@@ -746,64 +660,6 @@ def _init_scheduled_db():
PRIMARY KEY (owner, account_key, folder, message_key)
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS email_message_index (
owner TEXT NOT NULL DEFAULT '',
account_key TEXT NOT NULL DEFAULT '',
folder TEXT NOT NULL,
uid TEXT NOT NULL,
message_id TEXT,
subject TEXT,
from_name TEXT,
from_address TEXT,
to_text TEXT,
cc_text TEXT,
date_iso TEXT,
date_display TEXT,
date_epoch REAL DEFAULT 0,
size INTEGER DEFAULT 0,
flags TEXT DEFAULT '',
has_attachments INTEGER DEFAULT 0,
updated_at TEXT NOT NULL,
PRIMARY KEY (owner, account_key, folder, uid)
)
""")
conn.execute("""
CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
ON email_message_index(owner, account_key, folder, date_epoch DESC)
""")
conn.execute("""
CREATE INDEX IF NOT EXISTS ix_email_message_index_message_id
ON email_message_index(owner, account_key, message_id)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS email_body_preview_cache (
owner TEXT NOT NULL DEFAULT '',
account_key TEXT NOT NULL DEFAULT '',
folder TEXT NOT NULL,
uid TEXT NOT NULL,
message_id TEXT,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (owner, account_key, folder, uid)
)
""")
conn.execute("""
CREATE INDEX IF NOT EXISTS ix_email_body_preview_message_id
ON email_body_preview_cache(owner, account_key, message_id)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS email_attachment_metadata_cache (
owner TEXT NOT NULL DEFAULT '',
account_key TEXT NOT NULL DEFAULT '',
folder TEXT NOT NULL,
uid TEXT NOT NULL,
message_id TEXT,
attachments_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (owner, account_key, folder, uid)
)
""")
# Boundary cache — LLM-detected sig/quote start positions in the body.
# Stored as char offsets (-1 = no boundary found). Once cached, the
# client uses these to fold without ever re-calling the LLM.
@@ -923,13 +779,12 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
try:
if account_id:
row = db.query(_EA).filter(_EA.id == account_id, _EA.enabled == True).first() # noqa: E712
# If the resolved row isn't visible to this owner, treat as
# If the resolved row belongs to a different owner, treat as
# not-found rather than silently serving it. This is a defense
# in depth — `require_owner` already calls `_assert_owns_account`
# for query-param account_ids, but other callers (cookbook
# rules, scheduled poller) may not. Ownerless legacy rows are
# only visible on a mailbox match, same as the fallback below.
if row is not None and owner and not _account_visible_to_owner(row, owner):
# rules, scheduled poller) may not.
if row is not None and owner and row.owner and row.owner != owner:
row = None
# Fallback path — restrict to this owner's accounts so we don't
# leak another user's default mailbox to an unconfigured user.
@@ -1073,14 +928,6 @@ def _imap_connect(account_id: str | None = None, owner: str = "",
# `timeout` is overridable so short-lived callers (e.g. the service-health
# probe) can impose a tighter budget than the default IMAP timeout.
cfg = _get_email_config(account_id, owner=owner)
# Send-only (SMTP-only) account: no IMAP host means there is no inbox to
# read. Bail out with a clear, typed error instead of handing an empty
# host to imaplib — IMAP4("", 993) silently dials localhost:993 and fails
# with a confusing "[Errno 111] Connection refused" on every inbox poll.
if not cfg.get("imap_host"):
raise EmailNotConfiguredError(
f"IMAP is not configured for account {cfg.get('account_name') or 'default'!r}"
)
# Connection mode:
# STARTTLS on → plain + upgrade
# STARTTLS off + port 993 → implicit SSL (IMAPS)
@@ -1294,15 +1141,10 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str
try:
c = _imap_connect(account_id, owner=owner)
c.select(_q(src))
# Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy()
# and store() operate on message SEQUENCE NUMBERS, so addressing them
# with a UID moved/deleted the wrong message (or silently no-oped when
# the UID exceeded the message count). Use the UID commands, matching
# the move/delete path in email_routes.py.
status, _ = c.uid("COPY", uid, _q(dest))
status, _ = c.copy(uid, _q(dest))
if status != "OK":
return False
c.uid("STORE", uid, "+FLAGS", "\\Deleted")
c.store(uid, "+FLAGS", "\\Deleted")
c.expunge()
return True
except Exception as e:
@@ -1391,95 +1233,56 @@ def _list_attachments_from_msg(msg):
return attachments
idx = 0
for part in msg.walk():
if part.is_multipart():
continue
cd = str(part.get("Content-Disposition", ""))
ct = part.get_content_type()
is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename())
if part.is_multipart() and not is_attached_email:
continue
# Skip text/html body parts (only consider real attachments)
if ct in ("text/plain", "text/html") and "attachment" not in cd:
continue
filename = part.get_filename()
if filename:
filename = _decode_header(filename)
if ct == "message/rfc822" and not re.search(r"\.[A-Za-z0-9]{1,8}$", filename):
filename = f"{filename}.eml"
else:
# Inline images, etc. - generate a name
ext = "eml" if ct == "message/rfc822" else (ct.split("/")[-1] if "/" in ct else "bin")
ext = ct.split("/")[-1] if "/" in ct else "bin"
filename = f"attachment_{idx}.{ext}"
payload = part.get_payload(decode=True)
if payload is None and ct == "message/rfc822":
try:
payload = part.as_bytes()
except Exception:
payload = b""
size = len(payload) if payload is not None else 0
content_id = (part.get("Content-ID") or "").strip().strip("<>")
size = len(payload) if payload else 0
attachments.append({
"index": idx,
"filename": filename,
"content_type": ct,
"size": size,
"is_inline": "inline" in cd.lower(),
"content_id": content_id,
})
idx += 1
return attachments
def _is_likely_signature_image_attachment(att: dict) -> bool:
"""Match the reader's inline signature/logo image filter."""
filename = str((att or {}).get("filename") or "").lower()
if not re.search(r"\.(png|jpe?g|gif|bmp|svg|webp)$", filename):
return False
size = int((att or {}).get("size") or 0)
if re.search(r"^image\d{3,}\.(png|jpe?g|gif)$", filename):
return True
if re.search(r"^(signature|logo|sig|footer|banner)[-_\d]*\.(png|jpe?g|gif|svg)$", filename):
return True
return 0 < size < 30 * 1024
def _has_visible_attachments(msg) -> bool:
"""Return True only for attachments the reader will render as chips."""
return any(
not _is_likely_signature_image_attachment(att)
for att in _list_attachments_from_msg(msg)
)
def _extract_attachment_to_disk(msg, index, target_dir):
"""Extract a specific attachment to disk and return the file path."""
if not msg.is_multipart():
return None
idx = 0
for part in msg.walk():
if part.is_multipart():
continue
cd = str(part.get("Content-Disposition", ""))
ct = part.get_content_type()
is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename())
if part.is_multipart() and not is_attached_email:
continue
if ct in ("text/plain", "text/html") and "attachment" not in cd:
continue
if idx == index:
filename = part.get_filename()
if filename:
filename = _decode_header(filename)
if ct == "message/rfc822" and not re.search(r"\.[A-Za-z0-9]{1,8}$", filename):
filename = f"{filename}.eml"
else:
ext = "eml" if ct == "message/rfc822" else (ct.split("/")[-1] if "/" in ct else "bin")
ext = ct.split("/")[-1] if "/" in ct else "bin"
filename = f"attachment_{idx}.{ext}"
# Sanitize
safe_name = re.sub(r"[^\w\s\-.]", "_", filename).strip()
payload = part.get_payload(decode=True)
if payload is None and ct == "message/rfc822":
try:
payload = part.as_bytes()
except Exception:
payload = b""
if payload is None:
if not payload:
return None
target_dir.mkdir(parents=True, exist_ok=True)
filepath = target_dir / safe_name
@@ -1861,10 +1664,6 @@ class SendEmailRequest(BaseModel):
attachments: Optional[List[str]] = None
# Which account to send from. None = default account.
account_id: Optional[str] = None
# Source message for replies. When present, /send marks this exact message
# answered after successful delivery so it leaves undone/reply-soon views.
source_uid: Optional[str] = None
source_folder: Optional[str] = None
# Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
odysseus_kind: Optional[str] = None
# If true, /send waits for SMTP + Sent append and returns the sent UID.
routes/email_pollers.py
+146 -233
View File
@@ -29,7 +29,7 @@ from datetime import datetime
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from src.task_endpoint import resolve_task_candidates, task_llm_call_async
from src.llm_core import llm_call_async
from routes.email_helpers import (
_strip_think, _extract_reply, _apply_email_style_mechanics, _load_settings, _save_settings, _get_email_config,
@@ -44,46 +44,6 @@ from routes.email_helpers import (
logger = logging.getLogger(__name__)
# Recovers a `[{"action": ...}, ...]` JSON array from raw LLM output when the
# fenced-block strip leaves nothing usable. Runs on model output influenced by
# untrusted email bodies, so it must not backtrack: the object content class is
# `[^{}]` (brace-delimited, greedy) rather than the old `[^[\]]*?` lazy runs,
# which exploded exponentially on inputs like `[{"action"},{` + `}},{{` * N
# (CodeQL py/redos #198).
_CAL_ACTION_ARRAY_RE = re.compile(
r'\[\s*\{[^{}]*"action"[^{}]*\}\s*(?:,\s*\{[^{}]*\}\s*)*\]',
re.DOTALL,
)
def _extract_json_array_from_text(text: str):
"""Return the last valid JSON array embedded in model output, if any."""
if not text:
return None
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip(), flags=re.MULTILINE).strip()
decoder = json.JSONDecoder()
try:
parsed = decoder.decode(cleaned)
if isinstance(parsed, list):
return parsed
except Exception:
pass
# Models often explain themselves and finish with `[]` or `[{"action":...}]`.
# Scan every array opener and keep the last complete JSON array, rather than
# using a greedy regex that can swallow prose containing square brackets.
last = None
for idx, ch in enumerate(cleaned):
if ch != "[":
continue
try:
parsed, _end = decoder.raw_decode(cleaned[idx:])
except Exception:
continue
if isinstance(parsed, list):
last = parsed
return last
def _owner_for_email_account(account_id: str | None) -> str:
if not account_id:
@@ -117,8 +77,6 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
do_tag: bool = False, do_spam: bool = False,
do_calendar: bool = False,
days_back: int = 1,
account_id: str | None = None,
max_process: int | None = None,
progress_cb=None) -> str:
"""One iteration of the email scan. Temporarily flips settings flags
so the existing background-loop logic runs exactly once for the requested ops."""
@@ -133,12 +91,7 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
settings["email_auto_calendar"] = bool(do_calendar)
_save_settings(settings)
try:
return await _auto_summarize_pass(
days_back=days_back,
account_id=account_id,
max_process=max_process,
progress_cb=progress_cb,
)
return await _auto_summarize_pass(days_back=days_back, progress_cb=progress_cb)
finally:
s2 = _load_settings()
for k, v in prev.items():
@@ -176,7 +129,7 @@ def _latest_inbox_fallback_uids(conn, reconnect):
return [], reconnect()
async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
"""Single pass of the auto-summarize/reply scan.
When account_id is None, iterates over every enabled account in
@@ -203,41 +156,28 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
names = {}
if len(ids) <= 1:
# Single-account (or zero rows — fallback to legacy settings.json lookup)
return await _auto_summarize_pass_single(
days_back=days_back,
account_id=(ids[0] if ids else None),
max_process=max_process,
progress_cb=progress_cb,
)
return await _auto_summarize_pass_single(days_back=days_back, account_id=(ids[0] if ids else None), progress_cb=progress_cb)
outs = []
for idx, aid in enumerate(ids, start=1):
try:
await _emit_progress(progress_cb, f"{names.get(aid, aid[:8])}: starting ({idx}/{len(ids)})")
result = await _auto_summarize_pass_single(
days_back=days_back,
account_id=aid,
max_process=max_process,
progress_cb=progress_cb,
)
result = await _auto_summarize_pass_single(days_back=days_back, account_id=aid, progress_cb=progress_cb)
outs.append(f"[{names.get(aid, aid[:8])}] {result}")
except Exception as e:
logger.warning(f"auto-summarize pass failed for account {aid}: {e}")
outs.append(f"[{names.get(aid, aid[:8])}] error: {e}")
return "\n".join(outs)
return await _auto_summarize_pass_single(
days_back=days_back,
account_id=account_id,
max_process=max_process,
progress_cb=progress_cb,
)
return await _auto_summarize_pass_single(days_back=days_back, account_id=account_id, progress_cb=progress_cb)
async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
"""Single pass of the auto-summarize/reply scan for ONE account.
Reads current settings flags."""
import asyncio
import sqlite3 as _sql3
from src.llm_core import _uses_max_completion_tokens
import requests as _req
from src.endpoint_resolver import resolve_endpoint
from src.llm_core import _uses_max_completion_tokens, _restricts_temperature
settings = _load_settings()
auto_sum = settings.get("email_auto_summarize", False)
@@ -314,15 +254,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
).fetchall()}
if auto_tag or auto_spam:
if account_owner:
_tag_existing = {r[0] for r in _c.execute(
"SELECT message_id FROM email_tags WHERE owner=? AND (account_id=? OR account_id='' OR account_id IS NULL)",
(account_owner, account_id or ""),
).fetchall()}
_tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner=?", (account_owner,)).fetchall()}
else:
_tag_existing = {r[0] for r in _c.execute(
"SELECT message_id FROM email_tags WHERE (owner='' OR owner IS NULL) AND (account_id=? OR account_id='' OR account_id IS NULL)",
(account_id or "",),
).fetchall()}
_tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner='' OR owner IS NULL").fetchall()}
else:
_tag_existing = set()
_cal_existing = {r[0] for r in _c.execute(
@@ -351,10 +285,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
if auto_spam and not spam_folder:
logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move")
task_candidates = resolve_task_candidates(owner=account_owner)
if not task_candidates:
url, model, headers = resolve_endpoint("utility", owner=account_owner)
if not url:
url, model, headers = resolve_endpoint("default", owner=account_owner)
if not url or not model:
return "No model configured"
url, model, headers = task_candidates[0]
writing_style = settings.get("email_writing_style", "")
processed = 0
@@ -368,14 +303,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
_reply_failed = 0
_detail_lines = []
_current_folder = "INBOX"
# Calendar extraction is sequential and each row can involve a model
# call plus a calendar write. Keep the scheduled calendar-only pass
# below the 5-minute action budget instead of timing out mid-run.
_default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5
try:
_max_process = max(1, int(max_process)) if max_process is not None else _default_max_process
except Exception:
_max_process = _default_max_process
_max_process = 5
for _entry in uid_list:
if processed >= _max_process:
break
@@ -467,30 +395,48 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
req_headers.update(headers)
if need_sum:
tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
payload = {
"model": model,
"messages": [
{"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
{"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
],
tok_key: 16384,
"temperature": 0.3,
"stream": False,
}
# Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
if _restricts_temperature(model):
payload.pop("temperature", None)
try:
summary = await task_llm_call_async(
messages=[
{"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
{"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.3, max_tokens=16384, timeout=240,
# Use to_thread so this sync HTTP call doesn't freeze
# the entire event loop while the LLM thinks (240s).
resp = await asyncio.to_thread(
_req.post, url, json=payload, headers=req_headers, timeout=240
)
summary = _extract_reply((summary or "").strip())
if summary:
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_summaries
(message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_sum_existing.add(message_id)
_summaries_created += 1
_uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
_detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
if resp.ok:
rdata = resp.json()
m = (rdata.get("choices") or [{}])[0].get("message", {})
summary = (m.get("content") or "").strip()
summary = _extract_reply(summary)
if not summary:
rc = (m.get("reasoning_content") or "").strip()
bullets = [ln.strip() for ln in rc.split("\n") if re.match(r"^[-•*]\s+|^\d+[.)]\s+", ln.strip())]
summary = "\n".join(bullets) if bullets else ""
if summary:
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_summaries
(message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_sum_existing.add(message_id)
_summaries_created += 1
_uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
_detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
except Exception as e:
_uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
_detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
@@ -511,14 +457,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
if context_snippets:
sys_prompt += "\n\nRELEVANT CONTEXT FROM PAST EMAILS AND CONTACTS:\n" + "\n\n---\n\n".join(context_snippets[:5])
try:
reply = await task_llm_call_async(
reply = await llm_call_async(
url=url, model=model,
messages=[
{"role": "system", "content": sys_prompt},
{"role": "user", "content": f"Original email:\nFrom: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\nDraft a reply. Return only the reply body text."},
],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.7, max_tokens=1024, timeout=90,
temperature=0.7, max_tokens=1024,
headers=req_headers, timeout=90,
)
reply = _apply_email_style_mechanics(_extract_reply(reply or ""))
if reply:
@@ -545,8 +491,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
# ── Calendar event extraction (independent of reply drafting) ──
if need_cal:
_cal_run_count = 0
_cal_event_uids = []
_cal_parse_ok = False
try:
# Pull a snapshot of upcoming events so the LLM can decide
# create vs update vs cancel based on what already exists.
@@ -555,7 +499,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
_existing_summary = get_upcoming_events(_acct_owner, horizon_days=60, limit=40)
existing_json = json.dumps(_existing_summary)
is_sent = _folder.lower().startswith("sent") or "sent" in _folder.lower()
cal_extract = await task_llm_call_async(
cal_extract = await llm_call_async(
url=url, model=model,
messages=[
{"role": "system", "content": (
"You are a calendar assistant. The user receives emails AND sends replies "
@@ -606,22 +551,21 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
f"{body[:4000]}"
)},
],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.1, max_tokens=16384, timeout=75,
temperature=0.1, max_tokens=16384,
headers=req_headers, timeout=180,
)
_raw_original = cal_extract or ""
cal_extract = _strip_think(_raw_original)
cal_extract = re.sub(r"^```(?:json)?\s*|\s*```$", "", cal_extract, flags=re.MULTILINE).strip()
if not cal_extract and _raw_original:
matches = list(_CAL_ACTION_ARRAY_RE.finditer(_raw_original))
matches = list(re.finditer(r'\[\s*\{[^[\]]*?"action"[^[\]]*?\}\s*(?:,\s*\{[^[\]]*?\}\s*)*\]', _raw_original, re.DOTALL))
if matches:
cal_extract = matches[-1].group()
logger.info(f"[cal-extract] uid={uid.decode() if isinstance(uid, bytes) else uid} folder={_folder} subj={subject[:50]!r} raw_len={len(cal_extract)} orig_len={len(_raw_original)} raw={cal_extract[:800]!r}")
ops = _extract_json_array_from_text(cal_extract)
if ops is not None:
jm = re.search(r'\[.*\]', cal_extract, re.DOTALL)
if jm:
try:
_cal_parse_ok = True
ops = json.loads(jm.group())
logger.info(f"[cal-extract] parsed {len(ops)} op(s)")
if isinstance(ops, list) and ops:
from src.tool_implementations import do_manage_calendar
@@ -651,8 +595,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
r = await do_manage_calendar(json.dumps(args), owner=_acct_owner)
if r.get("exit_code", 0) == 0:
logger.info(f"[cal-extract] Updated event uid={cuid} → {op.get('title')} {op['date']}")
if cuid and cuid not in _cal_event_uids:
_cal_event_uids.append(cuid)
_cal_run_count += 1
else:
logger.warning(f"[cal-extract] update failed: {r.get('error')}")
@@ -733,43 +675,28 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
r = await do_manage_calendar(cal_args, owner=_acct_owner)
if r.get("exit_code", 0) == 0:
logger.info(f"[cal-extract] Created event: {op['title']} on {op['date']}")
_created_uid = (r.get("uid") or "").strip()
if _created_uid and _created_uid not in _cal_event_uids:
_cal_event_uids.append(_created_uid)
_events_created += 1
_cal_run_count += 1
else:
logger.warning(f"[cal-extract] create failed: {r.get('error')} args={cal_args[:200]}")
except Exception as je:
logger.warning(f"[cal-extract] JSON parse failed: {je} on raw={cal_extract[:200]!r}")
else:
logger.warning(f"[cal-extract] no JSON array found on raw={cal_extract[:200]!r}")
except Exception as e:
logger.warning(f"[cal-extract] Meeting extraction LLM call failed for uid={uid}: {e}")
else:
# Record successfully parsed results so we don't re-LLM
# no-op emails. Transient LLM failures are retried on
# the next poll run.
try:
if _cal_parse_ok:
_cc = _sql3.connect(SCHEDULED_DB)
_cc.execute(
"INSERT OR REPLACE INTO email_calendar_extractions "
"(message_id, owner, uid, event_uids, events_created, created_at) VALUES (?, ?, ?, ?, ?, ?)",
(
message_id,
account_owner or "",
uid.decode() if isinstance(uid, bytes) else str(uid),
json.dumps(_cal_event_uids),
_cal_run_count,
datetime.utcnow().isoformat(),
),
)
_cc.commit()
_cc.close()
_cal_existing.add(message_id)
except Exception as ce:
logger.debug(f"Could not cache calendar extraction: {ce}")
# Record we processed this email so we don't re-LLM next run
try:
_cc = _sql3.connect(SCHEDULED_DB)
_cc.execute(
"INSERT OR REPLACE INTO email_calendar_extractions "
"(message_id, owner, uid, events_created, created_at) VALUES (?, ?, ?, ?, ?)",
(message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid),
_cal_run_count, datetime.utcnow().isoformat())
)
_cc.commit()
_cc.close()
_cal_existing.add(message_id)
except Exception as ce:
logger.debug(f"Could not cache calendar extraction: {ce}")
if need_urgent:
try:
@@ -801,11 +728,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
"temperature": 0,
tok_key: 200,
}
urg_raw = await task_llm_call_async(
messages=payload["messages"],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0, max_tokens=200, timeout=60,
urg_raw = await llm_call_async(
url=url, model=model, messages=payload["messages"],
temperature=0, max_tokens=200, headers=req_headers, timeout=60,
)
urg_raw = _strip_think(urg_raw or "")
urg_raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", urg_raw, flags=re.MULTILINE).strip()
@@ -906,13 +831,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
class_sys = (
"Classify the email. Return ONLY a JSON object, no prose, no markdown fences. "
"Schema: {\"tags\": [\"tag1\"], \"spam\": false, \"reason\": \"short\"}. "
"Pick 1-3 tags from: work, personal, urgent, action-needed, finance, bills, "
"receipt, legal, travel, newsletter, promo, notification, security, social, "
"shopping, calendar, support.\n\n"
"Use work for professional/company/client/operations messages. "
"Use personal for friends/family/private-life messages. "
"Use urgent for real time-sensitive consequences. "
"Use action-needed when the user likely needs to reply, pay, sign, book, or decide.\n\n"
"Pick 1-2 tags from: work, personal, finance, bills, receipt, travel, "
"newsletter, promo, notification, security, social, shopping, calendar.\n\n"
"Set spam=true for ANY of:\n"
"- Phishing, scams, chain mail, deceptive offers\n"
"- Marketing/promotional blasts (\"special offer\", \"limited time\", discount codes)\n"
@@ -929,55 +849,70 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
"If it's a mass-mailed generic update with no personal CTA, mark spam=true even if from a legitimate service. "
"Reason should be 5-10 words."
)
raw_out = await task_llm_call_async(
messages=[
tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
payload = {
"model": model,
"messages": [
{"role": "system", "content": class_sys},
{"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body[:4000]}"},
],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.1, max_tokens=512, timeout=120,
tok_key: 512,
"temperature": 0.1,
"stream": False,
}
# Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
if _restricts_temperature(model):
payload.pop("temperature", None)
# to_thread keeps the event loop responsive during the LLM call
resp = await asyncio.to_thread(
_req.post, url, json=payload, headers=req_headers, timeout=120
)
raw_out = _strip_think((raw_out or "").strip())
raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
parsed = None
if jm:
try:
parsed = json.loads(jm.group(0))
except Exception:
parsed = None
if parsed is not None:
_ALLOWED_TAGS = {"work","personal","urgent","action-needed","finance","bills",
"receipt","legal","travel","newsletter","marketing","notification",
"security","social","shopping","calendar","support"}
raw_tags = parsed.get("tags") or []
if isinstance(raw_tags, str):
raw_tags = [raw_tags]
tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
tags = ["marketing" if t == "promo" else t for t in tags]
tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
is_spam = bool(parsed.get("spam"))
spam_reason = str(parsed.get("reason") or "")[:200]
if not resp.ok:
logger.warning(f"Auto-classify {uid.decode() if isinstance(uid, bytes) else str(uid)} HTTP {resp.status_code}: {resp.text[:200]}")
else:
rdata = resp.json()
m = (rdata.get("choices") or [{}])[0].get("message", {})
raw_out = (m.get("content") or "").strip()
raw_out = _strip_think(raw_out)
raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
parsed = None
if jm:
try:
parsed = json.loads(jm.group(0))
except Exception:
parsed = None
if parsed is not None:
_ALLOWED_TAGS = {"work","personal","finance","bills","receipt","travel",
"newsletter","marketing","notification","security","social",
"shopping","calendar"}
raw_tags = parsed.get("tags") or []
if isinstance(raw_tags, str):
raw_tags = [raw_tags]
tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
tags = ["marketing" if t == "promo" else t for t in tags]
tags = [t for t in tags if t in _ALLOWED_TAGS][:2]
is_spam = bool(parsed.get("spam"))
spam_reason = str(parsed.get("reason") or "")[:200]
moved_to = ""
if is_spam and auto_spam and spam_folder:
if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
moved_to = spam_folder
logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
moved_to = ""
if is_spam and auto_spam and spam_folder:
if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
moved_to = spam_folder
logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_tags
(message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict,
spam_reason, moved_to, model_used, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", account_id or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender,
json.dumps(tags), 1 if is_spam else 0,
spam_reason, moved_to, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_tag_existing.add(message_id)
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_tags
(message_id, owner, uid, folder, subject, sender, tags, spam_verdict,
spam_reason, moved_to, model_used, created_at)
VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), subject, sender,
json.dumps(tags), 1 if is_spam else 0,
spam_reason, moved_to, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_tag_existing.add(message_id)
except Exception as e:
logger.warning(f"Auto-classify {uid} failed: {e}")
@@ -1068,28 +1003,6 @@ def _scheduled_poll_once() -> dict:
for r in rows:
sid = r[0]
try:
# Atomically claim this row before doing any work. Two
# pollers can race here (the in-process asyncio task and an
# externally cron-driven `odysseus-mail poll-scheduled`, or
# an admin running the CLI manually alongside the in-process
# one despite the ODYSSEUS_INPROCESS_POLLERS=0 guidance) -
# both can SELECT the same 'pending' row before either has
# updated its status. The UPDATE...WHERE status='pending' is
# the atomicity boundary: only the poller whose UPDATE
# actually changes a row (rowcount == 1) proceeds to send;
# a loser sees rowcount == 0 and skips it instead of sending
# a duplicate.
claim_conn = sqlite3.connect(SCHEDULED_DB)
claim_cur = claim_conn.execute(
"UPDATE scheduled_emails SET status='sending' WHERE id=? AND status='pending'",
(sid,),
)
claim_conn.commit()
claimed = claim_cur.rowcount == 1
claim_conn.close()
if not claimed:
continue
attachments = json.loads(r[8] or "[]")
row_account_id = r[9] if len(r) > 9 else None
odysseus_kind = r[10] if len(r) > 10 else "scheduled"
routes/email_routes.py
+252 -1961
View File
File diff suppressed because it is too large Load Diff
routes/embedding_routes.py
-1
View File
@@ -9,7 +9,6 @@ from pathlib import Path
from fastapi import APIRouter, HTTPException, Form, Depends
from core.constants import EMBEDDING_ENDPOINT_FILE, FASTEMBED_CACHE_DIR
from core.middleware import require_admin
from src.runtime_paths import get_app_root
logger = logging.getLogger(__name__)
routes/gallery/__init__.py
-6
View File
@@ -1,6 +0,0 @@
"""Gallery route domain package (slice 2a, #4082/#4071).
Contains gallery_routes.py and gallery_helpers.py, migrated from the flat
routes/ directory. Backward-compat shims at routes/gallery_routes.py and
routes/gallery_helpers.py re-export from here.
"""
routes/gallery/gallery_helpers.py
-145
View File
@@ -1,145 +0,0 @@
"""gallery_helpers.py — extracted helpers, models, and small utilities.
Imported by gallery_routes.py."""
"""Gallery routes — browsable library for photos and AI-generated images."""
import logging
from datetime import datetime
from typing import Dict, Any, Optional
from pydantic import BaseModel
from core.database import GalleryImage
from src.auth_helpers import _auth_disabled
logger = logging.getLogger(__name__)
# ---- Request schemas ----
class GalleryPatch(BaseModel):
tags: Optional[str] = None
favorite: Optional[bool] = None
album_id: Optional[str] = None
# ---- EXIF extraction ----
def _extract_exif(content: bytes) -> dict:
"""Extract EXIF metadata from image bytes. Returns dict of fields."""
result = {"width": None, "height": None}
try:
from PIL import Image
from io import BytesIO
img = Image.open(BytesIO(content))
# Read the raw EXIF before any transpose: exif_transpose strips the
# orientation tag and with it the parsed EXIF view.
exif = img._getexif() if hasattr(img, '_getexif') else None
# Record DISPLAY dimensions (EXIF-rotated), matching upload_handler.
# A phone photo with Orientation 6/8 is stored landscape but shown
# portrait, so the raw width/height swap the aspect ratio.
try:
from PIL import ImageOps
img = ImageOps.exif_transpose(img) or img
except Exception:
pass
result["width"] = img.width
result["height"] = img.height
if not exif:
return result
# EXIF tag IDs
# 271=Make, 272=Model, 306=DateTime, 36867=DateTimeOriginal
# 34853=GPSInfo
result["camera_make"] = str(exif.get(271, "")).strip() or None
result["camera_model"] = str(exif.get(272, "")).strip() or None
# Date taken
for tag_id in (36867, 36868, 306): # DateTimeOriginal, DateTimeDigitized, DateTime
raw = exif.get(tag_id)
if raw:
try:
result["taken_at"] = datetime.strptime(str(raw).strip(), "%Y:%m:%d %H:%M:%S")
break
except (ValueError, TypeError):
pass
# GPS
gps_info = exif.get(34853)
if gps_info and isinstance(gps_info, dict):
try:
def _to_deg(vals):
d, m, s = [float(v) for v in vals]
return d + m / 60 + s / 3600
if 2 in gps_info and 4 in gps_info:
lat = _to_deg(gps_info[2])
lng = _to_deg(gps_info[4])
if gps_info.get(1) == 'S': lat = -lat
if gps_info.get(3) == 'W': lng = -lng
result["gps_lat"] = f"{lat:.6f}"
result["gps_lng"] = f"{lng:.6f}"
except Exception:
pass
except Exception as e:
# User-visible failure (photo loses metadata): surface at WARNING
# and record on the result so the upload endpoint can pass it back.
logger.warning(f"EXIF extraction failed: {e}")
result["exif_error"] = str(e)
return result
# ---- Helpers ----
def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any]:
return {
"id": img.id,
"filename": img.filename,
"url": f"/api/generated-image/{img.filename}",
"prompt": img.prompt,
"caption": img.caption or "",
"model": img.model,
"size": img.size,
"quality": img.quality,
"tags": img.tags or "",
"ai_tags": img.ai_tags or "",
"user_tags": img.tags or "",
"session_id": img.session_id,
"session_name": session_name,
"album_id": img.album_id,
"is_active": img.is_active,
"favorite": img.favorite or False,
"taken_at": img.taken_at.isoformat() if img.taken_at else None,
"camera": f"{img.camera_make or ''} {img.camera_model or ''}".strip() or None,
"gps": {"lat": img.gps_lat, "lng": img.gps_lng} if img.gps_lat else None,
"width": img.width,
"height": img.height,
"file_size": img.file_size,
"created_at": img.created_at.isoformat() if img.created_at else None,
"updated_at": img.updated_at.isoformat() if img.updated_at else None,
}
def _owner_filter(q, user, model_cls=GalleryImage):
"""Apply owner filtering to a gallery query.
``get_current_user`` returns None both in auth-disabled single-user mode
and when auth is enabled but no current user was resolved. Preserve the
single-user behavior, but fail closed for auth-enabled null-user states.
"""
if user is not None:
return q.filter(model_cls.owner == user)
if _auth_disabled():
return q
return q.filter(False)
def _human_size(nbytes):
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if abs(nbytes) < 1024:
return f"{nbytes:.1f} {unit}"
nbytes /= 1024
return f"{nbytes:.1f} PB"
routes/gallery/gallery_routes.py
-1966
View File
File diff suppressed because it is too large Load Diff
routes/gallery_helpers.py
+140 -10
View File
@@ -1,14 +1,144 @@
"""Backward-compat shim - canonical location is routes/gallery/gallery_helpers.py.
"""gallery_helpers.py — extracted helpers, models, and small utilities.
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.gallery_helpers``, ``from routes.gallery_helpers import X``,
``importlib.import_module("routes.gallery_helpers")``, and
``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the same
object. Keeps existing import paths working after slice 2a (#4082/#4071).
"""
Imported by gallery_routes.py."""
import sys as _sys
"""Gallery routes — browsable library for photos and AI-generated images."""
from routes.gallery import gallery_helpers as _canonical # noqa: F401
import logging
from datetime import datetime
from typing import Dict, Any, Optional
_sys.modules[__name__] = _canonical
from pydantic import BaseModel
from core.database import GalleryImage
from src.auth_helpers import _auth_disabled
logger = logging.getLogger(__name__)
# ---- Request schemas ----
class GalleryPatch(BaseModel):
tags: Optional[str] = None
favorite: Optional[bool] = None
album_id: Optional[str] = None
# ---- EXIF extraction ----
def _extract_exif(content: bytes) -> dict:
"""Extract EXIF metadata from image bytes. Returns dict of fields."""
result = {"width": None, "height": None}
try:
from PIL import Image
from io import BytesIO
img = Image.open(BytesIO(content))
# Read the raw EXIF before any transpose: exif_transpose strips the
# orientation tag and with it the parsed EXIF view.
exif = img._getexif() if hasattr(img, '_getexif') else None
# Record DISPLAY dimensions (EXIF-rotated), matching upload_handler.
# A phone photo with Orientation 6/8 is stored landscape but shown
# portrait, so the raw width/height swap the aspect ratio.
try:
from PIL import ImageOps
img = ImageOps.exif_transpose(img) or img
except Exception:
pass
result["width"] = img.width
result["height"] = img.height
if not exif:
return result
# EXIF tag IDs
# 271=Make, 272=Model, 306=DateTime, 36867=DateTimeOriginal
# 34853=GPSInfo
result["camera_make"] = str(exif.get(271, "")).strip() or None
result["camera_model"] = str(exif.get(272, "")).strip() or None
# Date taken
for tag_id in (36867, 36868, 306): # DateTimeOriginal, DateTimeDigitized, DateTime
raw = exif.get(tag_id)
if raw:
try:
result["taken_at"] = datetime.strptime(str(raw).strip(), "%Y:%m:%d %H:%M:%S")
break
except (ValueError, TypeError):
pass
# GPS
gps_info = exif.get(34853)
if gps_info and isinstance(gps_info, dict):
try:
def _to_deg(vals):
d, m, s = [float(v) for v in vals]
return d + m / 60 + s / 3600
if 2 in gps_info and 4 in gps_info:
lat = _to_deg(gps_info[2])
lng = _to_deg(gps_info[4])
if gps_info.get(1) == 'S': lat = -lat
if gps_info.get(3) == 'W': lng = -lng
result["gps_lat"] = f"{lat:.6f}"
result["gps_lng"] = f"{lng:.6f}"
except Exception:
pass
except Exception as e:
# User-visible failure (photo loses metadata): surface at WARNING
# and record on the result so the upload endpoint can pass it back.
logger.warning(f"EXIF extraction failed: {e}")
result["exif_error"] = str(e)
return result
# ---- Helpers ----
def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any]:
return {
"id": img.id,
"filename": img.filename,
"url": f"/api/generated-image/{img.filename}",
"prompt": img.prompt,
"model": img.model,
"size": img.size,
"quality": img.quality,
"tags": img.tags or "",
"ai_tags": img.ai_tags or "",
"user_tags": img.tags or "",
"session_id": img.session_id,
"session_name": session_name,
"album_id": img.album_id,
"is_active": img.is_active,
"favorite": img.favorite or False,
"taken_at": img.taken_at.isoformat() if img.taken_at else None,
"camera": f"{img.camera_make or ''} {img.camera_model or ''}".strip() or None,
"gps": {"lat": img.gps_lat, "lng": img.gps_lng} if img.gps_lat else None,
"width": img.width,
"height": img.height,
"file_size": img.file_size,
"created_at": img.created_at.isoformat() if img.created_at else None,
"updated_at": img.updated_at.isoformat() if img.updated_at else None,
}
def _owner_filter(q, user, model_cls=GalleryImage):
"""Apply owner filtering to a gallery query.
``get_current_user`` returns None both in auth-disabled single-user mode
and when auth is enabled but no current user was resolved. Preserve the
single-user behavior, but fail closed for auth-enabled null-user states.
"""
if user is not None:
return q.filter(model_cls.owner == user)
if _auth_disabled():
return q
return q.filter(False)
def _human_size(nbytes):
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
if abs(nbytes) < 1024:
return f"{nbytes:.1f} {unit}"
nbytes /= 1024
return f"{nbytes:.1f} PB"
routes/gallery_routes.py
+1930 -12
View File
File diff suppressed because it is too large Load Diff
routes/history/__init__.py
-5
View File
@@ -1,5 +0,0 @@
"""History route domain package (slice 2d, #4082/#4071).
Contains history_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/history_routes.py re-exports from here.
"""
routes/history/history_routes.py
-794
View File
@@ -1,794 +0,0 @@
"""History routes — session history, truncation, fork, conversation topics."""
import json
import uuid
import logging
import re
from typing import Dict, Any, Optional
from fastapi import APIRouter, Request, HTTPException
from core.models import ChatMessage
from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
from src.auth_helpers import effective_user
from src.topic_analyzer import analyze_topics
from src.upload_handler import reserve_message_upload_references
from routes.session_routes import (
_message_role,
_message_text,
_reject_compact_during_active_run,
_verify_session_owner,
)
logger = logging.getLogger(__name__)
_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
def _history_display_content(content: Any) -> Any:
"""Return a lightweight browser-display copy of stored message content.
Older multimodal user messages may be persisted as a JSON *string*
containing image_url blocks with inline base64 image bytes. Those bytes are
needed for model calls when the turn is first sent, but they should not be
sent back through /api/history every time the user opens the chat. The
attachment metadata already carries file ids/names for the UI cards.
"""
if isinstance(content, list):
text_parts = []
omitted_media = 0
for block in content:
if not isinstance(block, dict):
continue
if block.get("type") == "text":
text = block.get("text")
if isinstance(text, str) and text:
text_parts.append(text)
elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
omitted_media += 1
text = "\n".join(text_parts).strip()
if omitted_media and not text:
return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
return text
if not isinstance(content, str):
return content
if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
return content
stripped = content.lstrip()
if stripped.startswith("["):
try:
blocks = json.loads(content)
except (json.JSONDecodeError, TypeError, ValueError):
blocks = None
if isinstance(blocks, list):
text_parts = []
for block in blocks:
if not isinstance(block, dict):
continue
if block.get("type") == "text":
text = block.get("text")
if isinstance(text, str) and text:
text_parts.append(text)
if text_parts:
return "\n".join(text_parts).strip()
if "data:image/" in content:
return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
return content
def _merge_continue_rows_to_delete(db_messages, db1, db2):
"""DB rows to delete when merging the last two assistant messages.
Always the second assistant message (db2), plus ONLY the single
intervening "continue" user message (the one carrying "previous response
was interrupted") — matching the in-memory merge. The previous code
deleted the whole index range between the two assistant rows, destroying
any tool/system/user messages in between and desyncing the DB from the
in-memory history.
"""
to_delete = [db2]
i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
if i1 is not None and i2 is not None and i2 - 1 > i1:
between = db_messages[i2 - 1]
if getattr(between, "role", "") == "user" and "previous response was interrupted" in (getattr(between, "content", "") or ""):
to_delete.append(between)
return to_delete
def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
router = APIRouter(tags=["history"])
def _reserve_message_uploads(
request: Request,
content: Any,
metadata: Any = None,
) -> None:
try:
missing_id = reserve_message_upload_references(
upload_handler,
effective_user(request),
content,
metadata,
)
except (TypeError, ValueError) as exc:
raise HTTPException(400, "Invalid message attachment metadata") from exc
if missing_id:
raise HTTPException(
409,
f"Referenced upload is no longer available: {missing_id}",
)
def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
entry = {"role": m.role, "content": _history_display_content(m.content)}
meta = {}
if m.meta_data:
try:
meta = json.loads(m.meta_data) or {}
except (json.JSONDecodeError, ValueError):
meta = {}
if m.timestamp and "timestamp" not in meta:
meta["timestamp"] = m.timestamp.isoformat() + "Z"
if meta:
entry["metadata"] = meta
return entry
@router.get("/api/history/{session_id}")
async def get_session_history(
request: Request,
session_id: str,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Dict[str, Any]:
_verify_session_owner(request, session_id)
if limit is not None:
page_limit = max(1, min(int(limit), 100))
db = SessionLocal()
try:
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session is None:
raise HTTPException(404, f"Session '{session_id}' not found")
total = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.count()
)
page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
page_offset = max(0, min(page_offset, total))
rows = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.offset(page_offset)
.limit(page_limit)
.all()
)
history_dict = [
entry for entry in (_db_history_entry(m) for m in rows)
if not (entry.get("metadata") or {}).get("hidden")
]
return {
"history": history_dict,
"model": db_session.model,
"endpoint_url": db_session.endpoint_url,
"name": db_session.name,
"offset": page_offset,
"limit": page_limit,
"total": total,
"has_more_before": page_offset > 0,
"has_more_after": page_offset + len(rows) < total,
}
finally:
db.close()
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, f"Session '{session_id}' not found")
history_dict = []
for msg in session.history:
if isinstance(msg, ChatMessage):
# Skip hidden messages (e.g. compaction summaries for AI context)
if msg.metadata and msg.metadata.get("hidden"):
continue
entry = {"role": msg.role, "content": _history_display_content(msg.content)}
if msg.metadata:
entry["metadata"] = msg.metadata
history_dict.append(entry)
elif isinstance(msg, dict):
if msg.get("metadata", {}).get("hidden"):
continue
entry = {
"role": msg.get("role", ""),
"content": _history_display_content(msg.get("content", "")),
}
if msg.get("metadata"):
entry["metadata"] = msg["metadata"]
history_dict.append(entry)
# Fallback: load from DB if in-memory is empty
if not history_dict:
db = SessionLocal()
try:
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
db_history = []
for m in db_messages:
db_history.append(_db_history_entry(m))
if db_history:
# Rebuild in-memory history from the full set so hidden
# messages (e.g. compaction summaries) are kept for AI context.
session.history = [
ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
for m in db_history
]
# Response excludes hidden messages, matching the in-memory path.
history_dict = [
m for m in db_history
if not (m.get("metadata") or {}).get("hidden")
]
except Exception as e:
logger.error(f"DB fallback failed for {session_id}: {e}")
finally:
db.close()
return {
"history": history_dict,
"model": session.model,
"endpoint_url": session.endpoint_url,
"name": session.name,
}
@router.post("/api/session/{session_id}/truncate")
async def truncate_session(request: Request, session_id: str):
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
result = session_manager.truncate_messages(session_id, keep_count)
return {"status": "ok", "kept": keep_count, "truncated": result}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Truncate error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/message")
async def add_message(request: Request, session_id: str):
"""Add a message to a session (for slash command persistence)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
role = body.get("role", "assistant")
content = body.get("content", "")
if not content:
raise HTTPException(400, "content is required")
metadata = body.get("metadata")
_reserve_message_uploads(request, content, metadata)
msg = ChatMessage(role=role, content=content, metadata=metadata)
session_manager.add_message(session_id, msg)
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
@router.post("/api/session/{session_id}/delete-messages")
async def delete_messages(request: Request, session_id: str):
"""Delete specific messages by DB ID (or legacy index)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_ids = body.get("msg_ids", [])
indices = body.get("indices") # legacy fallback
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
if msg_ids:
# New ID-based delete
deleted = 0
for mid in msg_ids:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == mid,
DbChatMessage.session_id == session_id,
).first()
if db_msg:
db.delete(db_msg)
deleted += 1
# Remove from in-memory history by matching _db_id
def _get_db_id(m):
meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
return meta.get('_db_id') if isinstance(meta, dict) else None
session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
elif indices:
# Legacy index-based delete
indices = sorted(indices, reverse=True)
db_messages = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
deleted = 0
for idx in indices:
if 0 <= idx < len(db_messages):
db.delete(db_messages[idx])
deleted += 1
if 0 <= idx < len(session.history):
session.history.pop(idx)
else:
return {"status": "ok", "deleted": 0}
session.message_count = len(session.history)
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
from datetime import datetime, timezone
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
return {"status": "ok", "deleted": deleted}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Delete messages error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/edit-message")
async def edit_message(request: Request, session_id: str):
"""Edit the content of a message by its database ID."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_id = body.get("msg_id")
content = body.get("content")
if not msg_id or content is None:
raise HTTPException(400, "msg_id and content are required")
_reserve_message_uploads(request, content)
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == msg_id,
DbChatMessage.session_id == session_id,
).first()
if not db_msg:
raise HTTPException(404, "Message not found")
db_msg.content = content
meta = {}
if db_msg.meta_data:
try: meta = json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta['edited'] = True
db_msg.meta_data = json.dumps(meta)
# Update in-memory history by matching _db_id
for hmsg in session.history:
hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
if isinstance(hmsg, ChatMessage):
hmsg.content = content
hmsg.metadata['edited'] = True
elif isinstance(hmsg, dict):
hmsg['content'] = content
hmsg['metadata']['edited'] = True
break
db.commit()
return {"status": "ok"}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except HTTPException:
raise
except Exception as e:
logger.error(f"Edit message error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/mark-stopped")
async def mark_stopped(request: Request, session_id: str):
"""Mark the last assistant message as stopped by user."""
_verify_session_owner(request, session_id)
try:
session = session_manager.get_session(session_id)
# Find last assistant message and add stopped metadata
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata['stopped'] = True
if not msg.metadata.get('model'):
msg.metadata['model'] = session.model
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata']['stopped'] = True
if not msg['metadata'].get('model'):
msg['metadata']['model'] = session.model
break
# Also update in DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_messages:
meta = {}
if db_messages.meta_data:
try:
meta = _json.loads(db_messages.meta_data)
except (json.JSONDecodeError, ValueError):
pass
meta['stopped'] = True
if not meta.get('model'):
meta['model'] = session.model
db_messages.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Mark stopped error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/update-last-meta")
async def update_last_meta(request: Request, session_id: str):
"""Merge metadata into the last assistant message (e.g. save variants)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
meta_update = body.get("metadata", {})
session = session_manager.get_session(session_id)
# Update in-memory
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata.update(meta_update)
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata'].update(meta_update)
break
# Update in DB
db = SessionLocal()
try:
import json as _json
db_msg = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_msg:
meta = {}
if db_msg.meta_data:
try: meta = _json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta.update(meta_update)
db_msg.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Update last meta error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/merge-last-assistant")
async def merge_last_assistant(request: Request, session_id: str):
"""Merge the last two assistant messages into one (for continue)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
separator = body.get("separator", "\n\n")
session = session_manager.get_session(session_id)
# Find last two assistant messages in-memory
ai_indices = []
for i, msg in enumerate(session.history):
role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
if role == 'assistant':
ai_indices.append(i)
if len(ai_indices) < 2:
return {"status": "ok", "merged": False}
idx1, idx2 = ai_indices[-2], ai_indices[-1]
msg1, msg2 = session.history[idx1], session.history[idx2]
content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
merged_content = content1 + separator + content2
# Merge metadata
meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
merged_meta = {**meta1, **meta2}
merged_meta.pop('stopped', None) # no longer stopped after continue
# Update first message, remove second
if isinstance(msg1, ChatMessage):
msg1.content = merged_content
msg1.metadata = merged_meta
else:
msg1['content'] = merged_content
msg1['metadata'] = merged_meta
# Also remove the hidden "continue" user message between them if present
# It's the message at idx2-1 if it's a user message with continue text
remove_indices = [idx2]
if idx2 - 1 > idx1:
between = session.history[idx2 - 1]
between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
if between_role == 'user' and 'previous response was interrupted' in between_content:
remove_indices.insert(0, idx2 - 1)
for ri in sorted(remove_indices, reverse=True):
session.history.pop(ri)
# Update DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
# Find last two assistant messages in DB
ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
if len(ai_db) >= 2:
(_, db1), (_, db2) = ai_db[-2], ai_db[-1]
db1.content = merged_content
db1.meta_data = _json.dumps(merged_meta)
# Mirror the in-memory deletion: remove the second assistant
# message and ONLY the "continue" user message between them
# (not arbitrary tool/system/user rows). The old
# range-delete destroyed every row between the two assistant
# messages, desyncing the DB from the in-memory history.
for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
db.delete(_row)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok", "merged": True}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Merge assistant error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/fork")
async def fork_session(request: Request, session_id: str):
"""Create a new session with messages copied up to keep_count."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
# Get the source session
source = session_manager.sessions.get(session_id)
if not source:
raise HTTPException(404, "Session not found")
# Create new session
new_id = str(uuid.uuid4())
fork_name = f"\u2ADD {source.name}"
new_session = session_manager.create_session(
session_id=new_id,
name=fork_name,
endpoint_url=source.endpoint_url,
model=source.model,
rag=False,
owner=getattr(source, 'owner', None),
)
# Copy messages up to keep_count
msgs_to_copy = source.history[:keep_count]
for msg in msgs_to_copy:
# Copy the metadata dict. Sharing it would let the fork's
# persistence (add_message -> _persist_message stamps
# _db_id/timestamp onto the dict) mutate the SOURCE session's
# in-memory messages, corrupting their _db_id and breaking
# edit/delete-by-id on the original conversation.
meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
new_session.add_message(ChatMessage(msg.role, msg.content, meta))
try:
from src.event_bus import fire_event
fire_event("session_created", getattr(source, 'owner', None))
except Exception:
logger.debug("session_created event dispatch failed", exc_info=True)
return {
"status": "ok",
"id": new_id,
"name": fork_name,
"kept": len(msgs_to_copy),
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Fork error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.get("/api/conversations/topics")
async def get_conversation_topics(request: Request) -> Dict[str, Any]:
from src.auth_helpers import require_user
user = require_user(request)
try:
return analyze_topics(session_manager, owner=user or None)
except Exception as e:
raise HTTPException(500, f"Topic analysis failed: {e}")
@router.post("/api/session/{session_id}/compact")
async def compact_session(request: Request, session_id: str):
"""Manually trigger context compaction for a session."""
_verify_session_owner(request, session_id)
from src.auth_helpers import effective_user
owner = effective_user(request)
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, "Session not found")
_reject_compact_during_active_run(session_id)
try:
from src.model_context import estimate_tokens, get_context_length
from src.llm_core import llm_call_async
from src.endpoint_resolver import resolve_endpoint
if len(session.history) < 6:
return {"status": "ok", "message": "Not enough messages to compact"}
ctx_len = get_context_length(session.endpoint_url, session.model)
messages_before = session.get_context_messages()
used_before = estimate_tokens(messages_before)
pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
msg_count_before = len(session.history)
# Keep only last 4 messages, summarize the rest
keep_count = 4
older = session.history[:-keep_count]
recent = session.history[-keep_count:]
# Build text to summarize
convo_text = "\n".join(
f"{_message_role(m).upper()}: "
f"{_message_text(m)[:2000]}"
for m in older
)
# Use utility model if available
util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
compact_url = util_url or session.endpoint_url
compact_model = util_model or session.model
compact_headers = util_headers if util_url else session.headers
from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
summary = await llm_call_async(
compact_url, compact_model,
[
{"role": "system", "content": sys_prompt},
{"role": "user", "content": convo_text},
],
temperature=0.2, max_tokens=1024,
headers=compact_headers, timeout=30,
)
# Replace session history: summary as system message + recent messages
# System message holds the full summary for AI context
system_summary = ChatMessage(
role="system",
content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
metadata={"compacted": True, "hidden": True},
)
# Visible assistant message just shows stats
summary_msg = ChatMessage(
role="assistant",
content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
metadata={"compacted": True, "messages_removed": len(older)},
)
new_history = [system_summary, summary_msg] + list(recent)
session.history = new_history
session.message_count = len(session.history)
logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
# Update DB: delete old messages, insert summary
db = SessionLocal()
try:
db_msgs = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
# Delete all but the last keep_count
for m in db_msgs[:-keep_count]:
db.delete(m)
# Insert system summary (hidden, for AI context) and visible summary
import json as _json
import uuid
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
db_sys_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="system",
content=system_summary.content,
meta_data=_json.dumps(system_summary.metadata),
timestamp=now,
)
db.add(db_sys_summary)
db_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="assistant",
content=summary_msg.content,
meta_data=_json.dumps(summary_msg.metadata),
timestamp=now,
)
db.add(db_summary)
# Update session record
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
finally:
db.close()
session_manager.save_sessions()
used_after = estimate_tokens(session.get_context_messages())
pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
return {
"status": "ok",
"message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
"before": pct_before,
"after": pct_after,
}
except Exception as e:
logger.error(f"Manual compact error {session_id}: {e}")
raise HTTPException(500, str(e))
return router
routes/history_routes.py
+658 -13
View File
@@ -1,17 +1,662 @@
"""Backward-compat shim — canonical location is routes/history/history_routes.py.
"""History routes — session history, truncation, fork, conversation topics."""
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.history_routes``, ``from routes.history_routes import X``,
``importlib.import_module("routes.history_routes")``, and the
``import ... as history_routes`` + ``monkeypatch.setattr(history_routes, ...)``
pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py
all operate on the *same* object the application actually uses. Keeps existing
import paths working after slice 2d (#4082/#4071). Source-introspection tests
read the canonical file by path.
"""
import json
import uuid
import logging
from typing import Dict, Any
import sys as _sys
from fastapi import APIRouter, Request, HTTPException
from routes.history import history_routes as _canonical # noqa: F401
from core.models import ChatMessage
from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
from src.topic_analyzer import analyze_topics
from routes.session_routes import (
_message_role,
_message_text,
_reject_compact_during_active_run,
_verify_session_owner,
)
_sys.modules[__name__] = _canonical
logger = logging.getLogger(__name__)
def _merge_continue_rows_to_delete(db_messages, db1, db2):
"""DB rows to delete when merging the last two assistant messages.
Always the second assistant message (db2), plus ONLY the single
intervening "continue" user message (the one carrying "previous response
was interrupted") — matching the in-memory merge. The previous code
deleted the whole index range between the two assistant rows, destroying
any tool/system/user messages in between and desyncing the DB from the
in-memory history.
"""
to_delete = [db2]
i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
if i1 is not None and i2 is not None and i2 - 1 > i1:
between = db_messages[i2 - 1]
if getattr(between, "role", "") == "user" and "previous response was interrupted" in (getattr(between, "content", "") or ""):
to_delete.append(between)
return to_delete
def setup_history_routes(session_manager) -> APIRouter:
router = APIRouter(tags=["history"])
@router.get("/api/history/{session_id}")
async def get_session_history(request: Request, session_id: str) -> Dict[str, Any]:
_verify_session_owner(request, session_id)
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, f"Session '{session_id}' not found")
history_dict = []
for msg in session.history:
if isinstance(msg, ChatMessage):
# Skip hidden messages (e.g. compaction summaries for AI context)
if msg.metadata and msg.metadata.get("hidden"):
continue
entry = {"role": msg.role, "content": msg.content}
if msg.metadata:
entry["metadata"] = msg.metadata
history_dict.append(entry)
elif isinstance(msg, dict):
if msg.get("metadata", {}).get("hidden"):
continue
entry = {
"role": msg.get("role", ""),
"content": msg.get("content", ""),
}
if msg.get("metadata"):
entry["metadata"] = msg["metadata"]
history_dict.append(entry)
# Fallback: load from DB if in-memory is empty
if not history_dict:
db = SessionLocal()
try:
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
import json as _json
db_history = []
for m in db_messages:
entry = {"role": m.role, "content": m.content}
meta = {}
if m.meta_data:
try:
meta = _json.loads(m.meta_data) or {}
except (json.JSONDecodeError, ValueError):
meta = {}
if m.timestamp and "timestamp" not in meta:
meta["timestamp"] = m.timestamp.isoformat() + "Z"
if meta:
entry["metadata"] = meta
db_history.append(entry)
if db_history:
# Rebuild in-memory history from the full set so hidden
# messages (e.g. compaction summaries) are kept for AI context.
session.history = [
ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
for m in db_history
]
# Response excludes hidden messages, matching the in-memory path.
history_dict = [
m for m in db_history
if not (m.get("metadata") or {}).get("hidden")
]
except Exception as e:
logger.error(f"DB fallback failed for {session_id}: {e}")
finally:
db.close()
return {
"history": history_dict,
"model": session.model,
"endpoint_url": session.endpoint_url,
"name": session.name,
}
@router.post("/api/session/{session_id}/truncate")
async def truncate_session(request: Request, session_id: str):
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
result = session_manager.truncate_messages(session_id, keep_count)
return {"status": "ok", "kept": keep_count, "truncated": result}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Truncate error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/message")
async def add_message(request: Request, session_id: str):
"""Add a message to a session (for slash command persistence)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
role = body.get("role", "assistant")
content = body.get("content", "")
if not content:
raise HTTPException(400, "content is required")
msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
session_manager.add_message(session_id, msg)
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
@router.post("/api/session/{session_id}/delete-messages")
async def delete_messages(request: Request, session_id: str):
"""Delete specific messages by DB ID (or legacy index)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_ids = body.get("msg_ids", [])
indices = body.get("indices") # legacy fallback
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
if msg_ids:
# New ID-based delete
deleted = 0
for mid in msg_ids:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == mid,
DbChatMessage.session_id == session_id,
).first()
if db_msg:
db.delete(db_msg)
deleted += 1
# Remove from in-memory history by matching _db_id
def _get_db_id(m):
meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
return meta.get('_db_id') if isinstance(meta, dict) else None
session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
elif indices:
# Legacy index-based delete
indices = sorted(indices, reverse=True)
db_messages = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
deleted = 0
for idx in indices:
if 0 <= idx < len(db_messages):
db.delete(db_messages[idx])
deleted += 1
if 0 <= idx < len(session.history):
session.history.pop(idx)
else:
return {"status": "ok", "deleted": 0}
session.message_count = len(session.history)
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
from datetime import datetime, timezone
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
return {"status": "ok", "deleted": deleted}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Delete messages error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/edit-message")
async def edit_message(request: Request, session_id: str):
"""Edit the content of a message by its database ID."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_id = body.get("msg_id")
content = body.get("content")
if not msg_id or content is None:
raise HTTPException(400, "msg_id and content are required")
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == msg_id,
DbChatMessage.session_id == session_id,
).first()
if not db_msg:
raise HTTPException(404, "Message not found")
db_msg.content = content
meta = {}
if db_msg.meta_data:
try: meta = json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta['edited'] = True
db_msg.meta_data = json.dumps(meta)
# Update in-memory history by matching _db_id
for hmsg in session.history:
hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
if isinstance(hmsg, ChatMessage):
hmsg.content = content
hmsg.metadata['edited'] = True
elif isinstance(hmsg, dict):
hmsg['content'] = content
hmsg['metadata']['edited'] = True
break
db.commit()
return {"status": "ok"}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except HTTPException:
raise
except Exception as e:
logger.error(f"Edit message error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/mark-stopped")
async def mark_stopped(request: Request, session_id: str):
"""Mark the last assistant message as stopped by user."""
_verify_session_owner(request, session_id)
try:
session = session_manager.get_session(session_id)
# Find last assistant message and add stopped metadata
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata['stopped'] = True
if not msg.metadata.get('model'):
msg.metadata['model'] = session.model
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata']['stopped'] = True
if not msg['metadata'].get('model'):
msg['metadata']['model'] = session.model
break
# Also update in DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_messages:
meta = {}
if db_messages.meta_data:
try:
meta = _json.loads(db_messages.meta_data)
except (json.JSONDecodeError, ValueError):
pass
meta['stopped'] = True
if not meta.get('model'):
meta['model'] = session.model
db_messages.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Mark stopped error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/update-last-meta")
async def update_last_meta(request: Request, session_id: str):
"""Merge metadata into the last assistant message (e.g. save variants)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
meta_update = body.get("metadata", {})
session = session_manager.get_session(session_id)
# Update in-memory
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata.update(meta_update)
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata'].update(meta_update)
break
# Update in DB
db = SessionLocal()
try:
import json as _json
db_msg = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_msg:
meta = {}
if db_msg.meta_data:
try: meta = _json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta.update(meta_update)
db_msg.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Update last meta error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/merge-last-assistant")
async def merge_last_assistant(request: Request, session_id: str):
"""Merge the last two assistant messages into one (for continue)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
separator = body.get("separator", "\n\n")
session = session_manager.get_session(session_id)
# Find last two assistant messages in-memory
ai_indices = []
for i, msg in enumerate(session.history):
role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
if role == 'assistant':
ai_indices.append(i)
if len(ai_indices) < 2:
return {"status": "ok", "merged": False}
idx1, idx2 = ai_indices[-2], ai_indices[-1]
msg1, msg2 = session.history[idx1], session.history[idx2]
content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
merged_content = content1 + separator + content2
# Merge metadata
meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
merged_meta = {**meta1, **meta2}
merged_meta.pop('stopped', None) # no longer stopped after continue
# Update first message, remove second
if isinstance(msg1, ChatMessage):
msg1.content = merged_content
msg1.metadata = merged_meta
else:
msg1['content'] = merged_content
msg1['metadata'] = merged_meta
# Also remove the hidden "continue" user message between them if present
# It's the message at idx2-1 if it's a user message with continue text
remove_indices = [idx2]
if idx2 - 1 > idx1:
between = session.history[idx2 - 1]
between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
if between_role == 'user' and 'previous response was interrupted' in between_content:
remove_indices.insert(0, idx2 - 1)
for ri in sorted(remove_indices, reverse=True):
session.history.pop(ri)
# Update DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
# Find last two assistant messages in DB
ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
if len(ai_db) >= 2:
(_, db1), (_, db2) = ai_db[-2], ai_db[-1]
db1.content = merged_content
db1.meta_data = _json.dumps(merged_meta)
# Mirror the in-memory deletion: remove the second assistant
# message and ONLY the "continue" user message between them
# (not arbitrary tool/system/user rows). The old
# range-delete destroyed every row between the two assistant
# messages, desyncing the DB from the in-memory history.
for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
db.delete(_row)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok", "merged": True}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Merge assistant error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/fork")
async def fork_session(request: Request, session_id: str):
"""Create a new session with messages copied up to keep_count."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
# Get the source session
source = session_manager.sessions.get(session_id)
if not source:
raise HTTPException(404, "Session not found")
# Create new session
new_id = str(uuid.uuid4())
fork_name = f"\u2ADD {source.name}"
new_session = session_manager.create_session(
session_id=new_id,
name=fork_name,
endpoint_url=source.endpoint_url,
model=source.model,
rag=False,
owner=getattr(source, 'owner', None),
)
# Copy messages up to keep_count
msgs_to_copy = source.history[:keep_count]
for msg in msgs_to_copy:
# Copy the metadata dict. Sharing it would let the fork's
# persistence (add_message -> _persist_message stamps
# _db_id/timestamp onto the dict) mutate the SOURCE session's
# in-memory messages, corrupting their _db_id and breaking
# edit/delete-by-id on the original conversation.
meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
new_session.add_message(ChatMessage(msg.role, msg.content, meta))
try:
from src.event_bus import fire_event
fire_event("session_created", getattr(source, 'owner', None))
except Exception:
logger.debug("session_created event dispatch failed", exc_info=True)
return {
"status": "ok",
"id": new_id,
"name": fork_name,
"kept": len(msgs_to_copy),
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Fork error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.get("/api/conversations/topics")
async def get_conversation_topics(request: Request) -> Dict[str, Any]:
from src.auth_helpers import require_user
user = require_user(request)
try:
return analyze_topics(session_manager, owner=user or None)
except Exception as e:
raise HTTPException(500, f"Topic analysis failed: {e}")
@router.post("/api/session/{session_id}/compact")
async def compact_session(request: Request, session_id: str):
"""Manually trigger context compaction for a session."""
_verify_session_owner(request, session_id)
from src.auth_helpers import effective_user
owner = effective_user(request)
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, "Session not found")
_reject_compact_during_active_run(session_id)
try:
from src.model_context import estimate_tokens, get_context_length
from src.llm_core import llm_call_async
from src.endpoint_resolver import resolve_endpoint
if len(session.history) < 6:
return {"status": "ok", "message": "Not enough messages to compact"}
ctx_len = get_context_length(session.endpoint_url, session.model)
messages_before = session.get_context_messages()
used_before = estimate_tokens(messages_before)
pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
msg_count_before = len(session.history)
# Keep only last 4 messages, summarize the rest
keep_count = 4
older = session.history[:-keep_count]
recent = session.history[-keep_count:]
# Build text to summarize
convo_text = "\n".join(
f"{_message_role(m).upper()}: "
f"{_message_text(m)[:2000]}"
for m in older
)
# Use utility model if available
util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
compact_url = util_url or session.endpoint_url
compact_model = util_model or session.model
compact_headers = util_headers if util_url else session.headers
from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
summary = await llm_call_async(
compact_url, compact_model,
[
{"role": "system", "content": sys_prompt},
{"role": "user", "content": convo_text},
],
temperature=0.2, max_tokens=1024,
headers=compact_headers, timeout=30,
)
# Replace session history: summary as system message + recent messages
# System message holds the full summary for AI context
system_summary = ChatMessage(
role="system",
content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
metadata={"compacted": True, "hidden": True},
)
# Visible assistant message just shows stats
summary_msg = ChatMessage(
role="assistant",
content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
metadata={"compacted": True, "messages_removed": len(older)},
)
new_history = [system_summary, summary_msg] + list(recent)
session.history = new_history
session.message_count = len(session.history)
logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
# Update DB: delete old messages, insert summary
db = SessionLocal()
try:
db_msgs = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
# Delete all but the last keep_count
for m in db_msgs[:-keep_count]:
db.delete(m)
# Insert system summary (hidden, for AI context) and visible summary
import json as _json
import uuid
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
db_sys_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="system",
content=system_summary.content,
meta_data=_json.dumps(system_summary.metadata),
timestamp=now,
)
db.add(db_sys_summary)
db_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="assistant",
content=summary_msg.content,
meta_data=_json.dumps(summary_msg.metadata),
timestamp=now,
)
db.add(db_summary)
# Update session record
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
finally:
db.close()
session_manager.save_sessions()
used_after = estimate_tokens(session.get_context_messages())
pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
return {
"status": "ok",
"message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
"before": pct_before,
"after": pct_after,
}
except Exception as e:
logger.error(f"Manual compact error {session_id}: {e}")
raise HTTPException(500, str(e))
return router
routes/hwfit_routes.py
+7 -123
View File
@@ -1,13 +1,8 @@
import json
import os
import re
import shlex
import subprocess
from copy import deepcopy
from fastapi import APIRouter, HTTPException
from core.platform_compat import run_ssh_command
from routes._validators import validate_remote_host, validate_ssh_port
@@ -112,73 +107,6 @@ def _apply_manual_hardware(system, manual_mode="", manual_gpu_count="", manual_v
return system
def _run_model_probe(host: str, ssh_port: str, cmd: str) -> str:
try:
if host:
r = run_ssh_command(
host,
ssh_port or None,
cmd,
timeout=15,
connect_timeout=5,
strict_host_key_checking=False,
text=True,
)
else:
r = subprocess.run(["bash", "-lc", cmd], capture_output=True, text=True, timeout=15)
if r.returncode == 0:
return (r.stdout or "").strip()
except Exception:
return ""
return ""
def _inspect_model_path(model_path: str, host: str = "", ssh_port: str = "") -> dict:
"""Read lightweight metadata from a local or SSH-visible HF model folder."""
path = (model_path or "").strip()
if not path or path.startswith(("http://", "https://")):
return {}
if not (path.startswith("/") or path.startswith("~")):
return {}
qpath = shlex.quote(path)
qconfig = shlex.quote(os.path.join(path, "config.json"))
out = {}
exists = _run_model_probe(host, ssh_port, f"test -d {qpath} && printf found || printf missing")
if exists != "found":
target = host or "local container"
out["model_probe_error"] = f"Model path is not visible on {target}: {path}"
return out
raw_config = _run_model_probe(host, ssh_port, f"test -f {qconfig} && sed -n '1,240p' {qconfig}")
if raw_config:
try:
cfg = json.loads(raw_config)
except Exception:
cfg = {}
for key in ("context_length", "max_position_embeddings", "n_ctx_train", "model_max_length", "max_seq_len"):
value = cfg.get(key)
if isinstance(value, (int, float)) and value > 0:
out["model_ctx_max"] = int(value)
break
else:
out["model_probe_error"] = f"config.json not found in model path: {path}"
size_cmd = (
f"find {qpath} -type f \\( -name '*.safetensors' -o -name '*.bin' -o -name '*.gguf' \\) "
"-printf '%s\\n' 2>/dev/null | awk '{s+=$1} END {if (s>0) printf \"%.6f\", s/1073741824}'"
)
weights = _run_model_probe(host, ssh_port, size_cmd)
try:
weights_gb = float(weights)
except Exception:
weights_gb = 0.0
if weights_gb > 0:
out["model_weights_gb"] = round(weights_gb, 3)
elif "model_probe_error" not in out:
out["model_probe_error"] = f"No model weight files found in: {path}"
return out
def setup_hwfit_routes():
router = APIRouter(prefix="/api/hwfit", tags=["hwfit"])
@@ -191,7 +119,7 @@ def setup_hwfit_routes():
return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
@router.get("/models")
def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
"""Rank LLM models against detected hardware and return scored results.
gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -200,17 +128,11 @@ def setup_hwfit_routes():
fresh=true bypasses the hardware-detection cache."""
from services.hwfit.hardware import detect_system
from services.hwfit.fit import rank_models
from services.hwfit.models import get_models, model_catalog_path, refresh_dynamic_catalogs
from services.hwfit.models import get_models, model_catalog_path
host, ssh_port = _validate_detection_target(host, ssh_port)
system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
if system.get("error"):
return {"system": system, "models": [], "error": system["error"]}
catalog_refresh = None
if refresh_catalog:
try:
catalog_refresh = refresh_dynamic_catalogs(force=True)
except Exception as e:
catalog_refresh = {"error": str(e)}
if not get_models():
return {
"system": system,
@@ -310,13 +232,10 @@ def setup_hwfit_routes():
rank_kwargs.pop("target_context", None)
rank_kwargs.pop("fit_only", None)
results = rank_models(system, **rank_kwargs)
payload = {"system": system, "models": results}
if catalog_refresh is not None:
payload["catalog_refresh"] = catalog_refresh
return payload
return {"system": system, "models": results}
@router.get("/profiles")
def get_serve_profiles(model: str = "", model_path: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):
def get_serve_profiles(model: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):
"""Compute llama.cpp serve profiles (Quality/Balanced/Speed) for `model`
against the detected hardware on `host` (or local). Returns concrete
flags (n_gpu_layers, n_cpu_moe, cache_type, ctx) the serve UI can apply.
@@ -341,23 +260,8 @@ def setup_hwfit_routes():
# "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct".
s = (s or "").lower().strip()
s = s.split("/")[-1] # drop org prefix
for suffix in ("-gguf", "_gguf", ".gguf", "gguf"):
if s.endswith(suffix):
s = s[: -len(suffix)]
break
cut_at = None
for idx, ch in enumerate(s):
if ch not in "-_." or idx + 1 >= len(s):
continue
suffix = s[idx + 1:]
if (
suffix in {"fp8", "bf16", "f16"}
or suffix.startswith(("awq", "gptq", "iq"))
or (suffix.startswith("q") and len(suffix) > 1 and suffix[1].isdigit())
):
cut_at = idx
if cut_at is not None:
s = s[:cut_at]
s = re.sub(r"[-_.]?gguf$", "", s) # drop trailing gguf marker
s = re.sub(r"[-_.](q\d[^/]*|iq\d[^/]*|fp8|bf16|f16|awq[^/]*|gptq[^/]*)$", "", s)
return s
m = catalog.get(model)
@@ -368,16 +272,8 @@ def setup_hwfit_routes():
if nn and (nn == want or want.endswith(nn) or nn.endswith(want)):
m = entry
break
path_meta = _inspect_model_path(model_path or model, host=host, ssh_port=ssh_port)
if m is None:
return {
"system": system,
"profiles": [],
"error": "model not in catalog",
"model_ctx_max": int(path_meta.get("model_ctx_max") or 0),
"model_weights_gb": float(path_meta.get("model_weights_gb") or 0),
"model_probe_error": path_meta.get("model_probe_error") or "",
}
return {"system": system, "profiles": [], "error": "model not in catalog"}
# Surface the model's trained context limit so the serve UI can clamp a
# user-typed context down to it (asking for ctx > n_ctx_train overflows
# and, with a quantized KV cache, can crash the GPU).
@@ -387,16 +283,6 @@ def setup_hwfit_routes():
if isinstance(v, (int, float)) and v > 0:
model_ctx_max = int(v)
break
path_ctx_max = int(path_meta.get("model_ctx_max") or 0)
if path_ctx_max > 0:
model_ctx_max = max(model_ctx_max, path_ctx_max)
model_weights_gb = float(path_meta.get("model_weights_gb") or 0)
if model_weights_gb <= 0:
for k in ("min_vram_gb", "required_gb", "size_gb", "recommended_ram_gb", "min_ram_gb"):
v = m.get(k)
if isinstance(v, (int, float)) and v > 0:
model_weights_gb = float(v)
break
return {
"system": system,
"profiles": compute_serve_profiles(
@@ -405,8 +291,6 @@ def setup_hwfit_routes():
serve_quant=(serve_quant or None),
),
"model_ctx_max": model_ctx_max,
"model_weights_gb": model_weights_gb,
"model_probe_error": path_meta.get("model_probe_error") or "",
}
@router.get("/image-models")
routes/memory/__init__.py
-5
View File
@@ -1,5 +0,0 @@
"""Memory route domain package (slice 2c, #4082/#4071).
Contains memory_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/memory_routes.py re-exports from here.
"""
routes/memory/memory_routes.py
-552
View File
@@ -1,552 +0,0 @@
# routes/memory_routes.py
from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
from typing import Dict, Any, Optional, List
import json
import os
import re
import tempfile
import time
from datetime import datetime
import logging
# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
# Strips one prefix per call so import-from-LLM-output doesn't leave the
# numbering inside the saved memory text. Bullet markers (-, *, •) are
# also peeled here for the same reason.
_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
def _strip_list_prefix(text: str) -> str:
if not text:
return text
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
from services.memory import MemoryManager
from core.session_manager import SessionManager
from src.request_models import MemoryAddRequest
from core.database import SessionLocal
from src.llm_core import llm_call_async
from services.memory.memory_extractor import audit_memories
from src.auth_helpers import get_current_user, require_user
from src.endpoint_resolver import resolve_endpoint
from src.task_endpoint import resolve_task_endpoint
from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
logger = logging.getLogger(__name__)
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
"""Set up memory-related routes."""
router = APIRouter(prefix="/api/memory", tags=["memory"])
def _owner(request: Request) -> Optional[str]:
return get_current_user(request)
def _assert_session_owner(session_obj, user):
"""SECURITY: 404 if the caller does not own this session.
SessionManager.get_session is NOT owner-scoped — it returns any
session by id. These routes accept a caller-supplied session id, so
without this gate a user could target another tenant's session and
leak their chat history, their session-scoped LLM credentials, or the
session title. Mirrors session_routes / webhook_routes ownership.
"""
if user is not None and getattr(session_obj, "owner", None) != user:
raise HTTPException(404, "Session not found")
def _verify_memory_owner(memory: dict, user: Optional[str]):
"""Raise 404 if user doesn't own this memory.
SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
allowed any user to read/edit/delete memories with an empty/null owner
field, which leaked legacy data across the multi-user deploy.
"""
if user is None:
return # Auth disabled
if memory.get("owner") != user:
raise HTTPException(404, "Memory not found")
@router.post("/debug")
def debug_memory_relevance(request: Request, query: str = Form(...)):
"""Debug which memories would be triggered for a query"""
user = _owner(request)
memories = memory_manager.load(owner=user)
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
return {
"query": query,
"total_memories": len(memories),
"relevant_count": len(relevant),
"relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
for m in relevant]
}
@router.post("/add", response_model=Dict[str, Any])
async def api_add_memory(
request: Request,
memory_data: Optional[MemoryAddRequest] = None
):
"""Add a new memory entry with optional category, source, and session reference."""
from src.auth_helpers import require_privilege
require_privilege(request, "can_manage_memory")
if memory_data is None:
form = await request.form()
memory_data = MemoryAddRequest(
text=form.get("text"),
category=form.get("category", "fact"),
source=form.get("source", "user"),
session_id=form.get("session_id")
)
user = _owner(request)
text = (memory_data.text or "").strip()
if not text:
raise HTTPException(400, "empty memory")
user_mem = memory_manager.load(owner=user)
if memory_manager.find_duplicates(text, user_mem):
return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
if memory_data.session_id:
try:
session_obj = session_manager.get_session(memory_data.session_id)
except KeyError:
raise HTTPException(404, "Session not found")
_assert_session_owner(session_obj, user)
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
if memory_data.session_id:
new_entry["session_id"] = memory_data.session_id
all_mem = memory_manager.load_all()
all_mem.append(new_entry)
memory_manager.save(all_mem)
# Sync vector index
if memory_vector and memory_vector.healthy:
memory_vector.add(new_entry["id"], text)
try:
from src.event_bus import fire_event
fire_event("memory_added", user)
except Exception:
logger.debug("memory_added event dispatch failed", exc_info=True)
return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
@router.get("")
def api_get_memory(request: Request):
"""Return all memory entries with their metadata."""
user = _owner(request)
return {"memory": memory_manager.load(owner=user)}
@router.post("/search")
def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
"""Search across all memories with optional filters."""
user = _owner(request)
memories = memory_manager.load(owner=user)
if session_id:
memories = [m for m in memories if m.get("session_id") == session_id]
if category:
memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
return {"memories": relevant, "total": len(relevant), "query": query}
@router.get("/timeline")
def memory_timeline(request: Request):
"""Get memories in chronological order with source session information."""
user = _owner(request)
memories = memory_manager.load(owner=user)
sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
results = []
for memory in sorted_memories:
if "timestamp" in memory:
try:
dt = datetime.fromtimestamp(memory["timestamp"])
memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
except (ValueError, OSError, OverflowError):
memory["timestamp_str"] = "Unknown"
else:
memory["timestamp_str"] = "Unknown"
session_id = memory.get("session_id")
if session_id and session_id in session_manager.sessions:
try:
session = session_manager.get_session(session_id)
if session:
_assert_session_owner(session, user)
memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
except KeyError:
memory["session_name"] = "Unknown"
except HTTPException as exc:
if exc.status_code != 404:
raise
memory["session_name"] = "Unknown"
else:
memory["session_name"] = "Unknown"
results.append(memory)
return {"timeline": results, "total": len(results)}
@router.get("/by-session/{session_id}")
def get_memory_by_session(request: Request, session_id: str):
"""Get all memories associated with a specific session."""
user = _owner(request)
try:
_session_obj = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, f"Session {session_id} not found")
_assert_session_owner(_session_obj, user)
memories = memory_manager.load(owner=user)
session_memories = [m for m in memories if m.get("session_id") == session_id]
session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
try:
session = session_manager.get_session(session_id)
session_name = session.name if session else f"Session {session_id[:6]}"
except KeyError:
session_name = f"Session {session_id[:6]}"
for memory in session_memories:
memory["session_name"] = session_name
return {
"session_id": session_id,
"session_name": session_name,
"memory_count": len(session_memories),
"memories": session_memories
}
@router.post("/extract")
async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
"""Analyze a session's chat history and return memory suggestions."""
require_user(request)
try:
sess = session_manager.get_session(session)
except KeyError:
raise HTTPException(404, "Session not found")
_assert_session_owner(sess, _owner(request))
system_msg = {
"role": "system",
"content": (
"You are a helpful assistant. Analyze the entire conversation history provided and extract any "
"useful factual statements, contacts, addresses, phone numbers, or other information that the user "
"might want to remember for future interactions. Return each piece of information as a JSON object "
"with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
"Only include information that is specific and likely to be useful later."
),
}
messages = [system_msg] + sess.get_context_messages()
t_url, t_model, t_headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
)
try:
suggestion_text = await llm_call_async(
t_url,
t_model,
messages,
temperature=0.2,
max_tokens=500,
headers=t_headers,
)
try:
suggestions = json.loads(suggestion_text)
if isinstance(suggestions, list):
suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
else:
suggestions = []
except json.JSONDecodeError:
suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
return {"suggestions": [s for s in suggestions if s]}
except Exception as e:
logger.error(f"LLM memory extraction failed (session {session}): {e}")
fallback = memory_manager.extract_memory_from_chat(sess.history, session)
return {"suggestions": [item["text"] for item in fallback]}
@router.post("/audit")
async def api_audit_memories(request: Request, session: str = Form(None)):
"""Deduplicate and consolidate memories via LLM.
Uses task/utility/default settings through the shared resolver, with
the active session as fallback when no task or utility model is set.
Returns before and after memory counts.
"""
user = _owner(request)
fallback_url = fallback_model = None
fallback_headers = None
if session:
try:
sess = session_manager.get_session(session)
_assert_session_owner(sess, user)
fallback_url = sess.endpoint_url
fallback_model = sess.model
fallback_headers = sess.headers
except KeyError:
pass
endpoint_url, model, headers = resolve_task_endpoint(
fallback_url, fallback_model, fallback_headers, owner=user
)
if not endpoint_url or not model:
raise HTTPException(400, "No default model configured — set one in Settings")
result = await audit_memories(
memory_manager,
memory_vector,
endpoint_url,
model,
headers,
owner=user,
)
if "error" in result and "before" not in result:
raise HTTPException(502, f"Audit failed: {result['error']}")
return {
"ok": "error" not in result,
"before": result.get("before", 0),
"after": result.get("after", 0),
"removed": result.get("before", 0) - result.get("after", 0),
# True when the audit skipped the LLM because nothing changed
# since the last tidy. Frontend already says "Already clean"
# for removed==0, so this is here for future use / debugging.
"already_tidy": bool(result.get("already_tidy")),
}
@router.post("/import")
async def import_memories_from_file(
request: Request,
session: str | None = Form(None),
file: UploadFile = File(...)
):
"""Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
from src.auth_helpers import require_privilege
require_privilege(request, "can_manage_memory")
endpoint_url = None
model = None
headers = {}
user = _owner(request)
if session:
try:
sess = session_manager.get_session(session)
_assert_session_owner(sess, user)
except KeyError:
sess = None
except HTTPException as exc:
if exc.status_code != 404:
raise
sess = None
if sess is None:
logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
else:
endpoint_url, model, headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=user
)
else:
endpoint_url, model, headers = resolve_task_endpoint(owner=user)
if not endpoint_url or not model:
raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
filename = file.filename or "upload"
_, ext = os.path.splitext(filename.lower())
allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
if ext not in allowed:
raise HTTPException(400, f"Unsupported file type: {ext}")
# Extract text based on file type
if ext == ".pdf":
from src.document_processor import _process_pdf
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
text = _process_pdf(tmp_path, owner=_owner(request))
finally:
os.unlink(tmp_path)
else:
try:
text = content.decode("utf-8")
except UnicodeDecodeError:
from charset_normalizer import detect
encoding = (detect(content) or {}).get("encoding") or "utf-8"
text = content.decode(encoding, errors="replace")
if not text.strip():
return {"suggestions": [], "message": "No readable content found"}
# Fast path: a .json upload that already looks like a memories export
# (list of {text, category, ...} dicts, or list of strings) round-trips
# directly without spending an LLM call to re-extract its own output.
# Without this, re-importing a memories.json from another account
# ran the file through the extractor, which often re-emitted the
# entries as a numbered list (and the numbering leaked into the
# `text` field).
if ext == ".json":
try:
parsed = json.loads(text)
except json.JSONDecodeError:
parsed = None
if isinstance(parsed, list) and parsed:
direct = []
for item in parsed:
if isinstance(item, dict) and item.get("text"):
direct.append({
"text": _strip_list_prefix(str(item["text"])),
"category": item.get("category") or "fact",
})
elif isinstance(item, str) and item.strip():
direct.append({
"text": _strip_list_prefix(item.strip()),
"category": "fact",
})
if direct:
return {"suggestions": direct, "filename": filename}
# Truncate very long documents
if len(text) > 15000:
text = text[:15000] + "\n[Truncated]"
# Send to LLM for memory extraction
import_prompt = (
"You are a memory extraction assistant. The user uploaded a document. "
"Analyze the text below and extract specific, useful facts — things like "
"names, preferences, jobs, locations, relationships, opinions, projects, "
"goals, contacts, or any other personal details worth remembering.\n\n"
"Rules:\n"
"- Each fact should be a short, self-contained statement\n"
"- Do NOT extract generic knowledge\n"
"- Focus on personal, memorable information\n"
"- If there are no useful facts, return an empty array\n\n"
"Return a JSON array of objects with 'text' and 'category' fields.\n"
"Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
"Return ONLY valid JSON, no markdown fences."
)
try:
raw = await llm_call_async(
endpoint_url,
model,
[
{"role": "system", "content": import_prompt},
{"role": "user", "content": f"Document: {filename}\n\n{text}"},
],
temperature=0.2,
max_tokens=2000,
headers=headers,
)
# Parse JSON
raw = raw.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
suggestions = json.loads(raw)
if isinstance(suggestions, list):
normalized = []
for s in suggestions:
if not s:
continue
if isinstance(s, dict):
s = dict(s)
if s.get("text"):
s["text"] = _strip_list_prefix(str(s["text"]))
normalized.append(s)
else:
normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
suggestions = normalized
else:
suggestions = []
return {"suggestions": suggestions, "filename": filename}
except json.JSONDecodeError:
# Fallback: split by lines, stripping any "1.", "2)" markdown-list
# numbering the model added so saved memories don't keep the prefix.
lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
except Exception as e:
logger.error(f"Memory import extraction failed: {e}")
raise HTTPException(502, f"LLM extraction failed: {str(e)}")
@router.post("/{memory_id}/pin")
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
"""Pin or unpin a memory. Pinned memories are always included in context."""
user = _owner(request)
all_mem = memory_manager.load_all()
for i, memory in enumerate(all_mem):
if memory["id"] == memory_id:
_verify_memory_owner(memory, user)
all_mem[i]["pinned"] = pinned
memory_manager.save(all_mem)
return {"ok": True, "pinned": pinned}
raise HTTPException(404, f"Memory item {memory_id} not found")
# Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
@router.get("/{memory_id}")
def get_memory_item(request: Request, memory_id: str):
"""Get a specific memory item by ID."""
user = _owner(request)
memories = memory_manager.load(owner=user)
for memory in memories:
if memory["id"] == memory_id:
return {"memory": memory}
raise HTTPException(404, "Memory not found")
@router.put("/{memory_id}")
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
"""Update an existing memory item with new text and optional category."""
user = _owner(request)
all_mem = memory_manager.load_all()
for i, memory in enumerate(all_mem):
if memory["id"] == memory_id:
_verify_memory_owner(memory, user)
all_mem[i]["text"] = text.strip()
if category:
all_mem[i]["category"] = category
all_mem[i]["timestamp"] = int(time.time())
memory_manager.save(all_mem)
# Sync vector index (remove old, add updated)
if memory_vector and memory_vector.healthy:
memory_vector.remove(memory_id)
memory_vector.add(memory_id, text.strip())
return {"ok": True, "message": "Memory updated successfully"}
raise HTTPException(404, f"Memory item {memory_id} not found")
@router.delete("/{memory_id}")
def delete_memory(request: Request, memory_id: str):
"""Delete a memory item by its ID."""
user = _owner(request)
all_mem = memory_manager.load_all()
# Find and verify ownership before deleting
target = next((m for m in all_mem if m["id"] == memory_id), None)
if not target:
raise HTTPException(404, f"Memory item {memory_id} not found")
_verify_memory_owner(target, user)
all_mem = [m for m in all_mem if m["id"] != memory_id]
memory_manager.save(all_mem)
# Sync vector index
if memory_vector and memory_vector.healthy:
memory_vector.remove(memory_id)
return {"ok": True, "message": "Memory deleted successfully"}
return router
routes/memory_routes.py
+573 -14
View File
@@ -1,18 +1,577 @@
"""Backward-compat shim — canonical location is routes/memory/memory_routes.py.
# routes/memory_routes.py
from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
from typing import Dict, Any, Optional, List
import json
import os
import re
import tempfile
import time
from datetime import datetime
import logging
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.memory_routes``, ``from routes.memory_routes import X``,
``importlib.import_module("routes.memory_routes")``, and
``monkeypatch.setattr(routes.memory_routes, "ATTR", ...)`` (used by
test_memory_routes_session_owner.py and test_memory_owner_isolation.py via
``import ... as mr`` + ``setattr(mr, ...)``) all operate on the *same* object
the application actually uses. Keeps existing import paths working after
slice 2c (#4082/#4071). Source-introspection tests read the canonical file
by path.
"""
# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
# Strips one prefix per call so import-from-LLM-output doesn't leave the
# numbering inside the saved memory text. Bullet markers (-, *, •) are
# also peeled here for the same reason.
_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
import sys as _sys
from routes.memory import memory_routes as _canonical # noqa: F401
def _strip_list_prefix(text: str) -> str:
if not text:
return text
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
_sys.modules[__name__] = _canonical
from services.memory import MemoryManager
from core.session_manager import SessionManager
from src.request_models import MemoryAddRequest
from core.database import SessionLocal
from src.llm_core import llm_call_async
from services.memory.memory_extractor import audit_memories
from src.auth_helpers import get_current_user, require_user
from src.endpoint_resolver import resolve_endpoint
from src.task_endpoint import resolve_task_endpoint
from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
logger = logging.getLogger(__name__)
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
"""Set up memory-related routes."""
router = APIRouter(prefix="/api/memory", tags=["memory"])
def _owner(request: Request) -> Optional[str]:
return get_current_user(request)
def _assert_session_owner(session_obj, user):
"""SECURITY: 404 if the caller does not own this session.
SessionManager.get_session is NOT owner-scoped — it returns any
session by id. These routes accept a caller-supplied session id, so
without this gate a user could target another tenant's session and
leak their chat history, their session-scoped LLM credentials, or the
session title. Mirrors session_routes / webhook_routes ownership.
"""
if user is not None and getattr(session_obj, "owner", None) != user:
raise HTTPException(404, "Session not found")
def _verify_memory_owner(memory: dict, user: Optional[str]):
"""Raise 404 if user doesn't own this memory.
SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
allowed any user to read/edit/delete memories with an empty/null owner
field, which leaked legacy data across the multi-user deploy.
"""
if user is None:
return # Auth disabled
if memory.get("owner") != user:
raise HTTPException(404, "Memory not found")
@router.post("/debug")
def debug_memory_relevance(request: Request, query: str = Form(...)):
"""Debug which memories would be triggered for a query"""
user = _owner(request)
memories = memory_manager.load(owner=user)
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
return {
"query": query,
"total_memories": len(memories),
"relevant_count": len(relevant),
"relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
for m in relevant]
}
@router.post("/add", response_model=Dict[str, Any])
async def api_add_memory(
request: Request,
memory_data: Optional[MemoryAddRequest] = None
):
"""Add a new memory entry with optional category, source, and session reference."""
from src.auth_helpers import require_privilege
require_privilege(request, "can_manage_memory")
if memory_data is None:
form = await request.form()
memory_data = MemoryAddRequest(
text=form.get("text"),
category=form.get("category", "fact"),
source=form.get("source", "user"),
session_id=form.get("session_id")
)
user = _owner(request)
text = (memory_data.text or "").strip()
if not text:
raise HTTPException(400, "empty memory")
user_mem = memory_manager.load(owner=user)
if memory_manager.find_duplicates(text, user_mem):
return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
if memory_data.session_id:
try:
session_obj = session_manager.get_session(memory_data.session_id)
except KeyError:
raise HTTPException(404, "Session not found")
_assert_session_owner(session_obj, user)
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
if memory_data.session_id:
new_entry["session_id"] = memory_data.session_id
all_mem = memory_manager.load_all()
all_mem.append(new_entry)
memory_manager.save(all_mem)
# Sync vector index
if memory_vector and memory_vector.healthy:
memory_vector.add(new_entry["id"], text)
try:
from src.event_bus import fire_event
fire_event("memory_added", user)
except Exception:
logger.debug("memory_added event dispatch failed", exc_info=True)
return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
@router.get("")
def api_get_memory(request: Request):
"""Return all memory entries with their metadata."""
user = _owner(request)
return {"memory": memory_manager.load(owner=user)}
@router.post("/search")
def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
"""Search across all memories with optional filters."""
user = _owner(request)
memories = memory_manager.load(owner=user)
if session_id:
memories = [m for m in memories if m.get("session_id") == session_id]
if category:
memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
return {"memories": relevant, "total": len(relevant), "query": query}
@router.get("/timeline")
def memory_timeline(request: Request):
"""Get memories in chronological order with source session information."""
user = _owner(request)
memories = memory_manager.load(owner=user)
sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
results = []
for memory in sorted_memories:
if "timestamp" in memory:
try:
dt = datetime.fromtimestamp(memory["timestamp"])
memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
except (ValueError, OSError, OverflowError):
memory["timestamp_str"] = "Unknown"
else:
memory["timestamp_str"] = "Unknown"
session_id = memory.get("session_id")
if session_id and session_id in session_manager.sessions:
try:
session = session_manager.get_session(session_id)
if session:
_assert_session_owner(session, user)
memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
except KeyError:
memory["session_name"] = "Unknown"
except HTTPException as exc:
if exc.status_code != 404:
raise
memory["session_name"] = "Unknown"
else:
memory["session_name"] = "Unknown"
results.append(memory)
return {"timeline": results, "total": len(results)}
@router.get("/by-session/{session_id}")
def get_memory_by_session(request: Request, session_id: str):
"""Get all memories associated with a specific session."""
user = _owner(request)
try:
_session_obj = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, f"Session {session_id} not found")
_assert_session_owner(_session_obj, user)
memories = memory_manager.load(owner=user)
session_memories = [m for m in memories if m.get("session_id") == session_id]
session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
try:
session = session_manager.get_session(session_id)
session_name = session.name if session else f"Session {session_id[:6]}"
except KeyError:
session_name = f"Session {session_id[:6]}"
for memory in session_memories:
memory["session_name"] = session_name
return {
"session_id": session_id,
"session_name": session_name,
"memory_count": len(session_memories),
"memories": session_memories
}
@router.post("/extract")
async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
"""Analyze a session's chat history and return memory suggestions."""
require_user(request)
try:
sess = session_manager.get_session(session)
except KeyError:
raise HTTPException(404, "Session not found")
_assert_session_owner(sess, _owner(request))
system_msg = {
"role": "system",
"content": (
"You are a helpful assistant. Analyze the entire conversation history provided and extract any "
"useful factual statements, contacts, addresses, phone numbers, or other information that the user "
"might want to remember for future interactions. Return each piece of information as a JSON object "
"with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
"Only include information that is specific and likely to be useful later."
),
}
messages = [system_msg] + sess.get_context_messages()
t_url, t_model, t_headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
)
try:
suggestion_text = await llm_call_async(
t_url,
t_model,
messages,
temperature=0.2,
max_tokens=500,
headers=t_headers,
)
try:
suggestions = json.loads(suggestion_text)
if isinstance(suggestions, list):
suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
else:
suggestions = []
except json.JSONDecodeError:
suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
return {"suggestions": [s for s in suggestions if s]}
except Exception as e:
logger.error(f"LLM memory extraction failed (session {session}): {e}")
fallback = memory_manager.extract_memory_from_chat(sess.history, session)
return {"suggestions": [item["text"] for item in fallback]}
@router.post("/audit")
async def api_audit_memories(request: Request, session: str = Form(None)):
"""Deduplicate and consolidate memories via LLM.
Uses the default model from settings, or falls back to a session's model.
Returns before and after memory counts.
"""
from routes.model_routes import _load_settings, _normalize_base, build_chat_url
from core.database import ModelEndpoint
import json as _json
endpoint_url = model = None
headers = {}
# Try utility model from settings first — memory audit is a background
# task and should prefer the lighter utility model over the main chat model.
from src.task_endpoint import resolve_task_endpoint
user = _owner(request)
t_url, t_model, t_headers = resolve_task_endpoint(owner=user)
if t_url and t_model:
endpoint_url, model, headers = t_url, t_model, t_headers
else:
# Fall back to default model if no task/utility model configured
settings = _load_settings()
ep_id = settings.get("default_endpoint_id", "")
default_model = settings.get("default_model", "")
if ep_id:
db = SessionLocal()
try:
ep = db.query(ModelEndpoint).filter(
ModelEndpoint.id == ep_id, ModelEndpoint.is_enabled == True
).first()
if ep:
base = _normalize_base(ep.base_url)
endpoint_url = build_chat_url(base)
model = default_model
if not model and ep.models:
try:
models = _json.loads(ep.models) if isinstance(ep.models, str) else ep.models
if models:
model = models[0]
except Exception:
pass
if ep.api_key:
headers = {"Authorization": f"Bearer {ep.api_key}"}
finally:
db.close()
# Fall back to session model if no default configured
if not endpoint_url and session:
try:
sess = session_manager.get_session(session)
_assert_session_owner(sess, _owner(request))
endpoint_url = sess.endpoint_url
model = sess.model
headers = sess.headers
except KeyError:
pass
if not endpoint_url or not model:
raise HTTPException(400, "No default model configured — set one in Settings")
user = _owner(request)
result = await audit_memories(
memory_manager,
memory_vector,
endpoint_url,
model,
headers,
owner=user,
)
if "error" in result and "before" not in result:
raise HTTPException(502, f"Audit failed: {result['error']}")
return {
"ok": "error" not in result,
"before": result.get("before", 0),
"after": result.get("after", 0),
"removed": result.get("before", 0) - result.get("after", 0),
# True when the audit skipped the LLM because nothing changed
# since the last tidy. Frontend already says "Already clean"
# for removed==0, so this is here for future use / debugging.
"already_tidy": bool(result.get("already_tidy")),
}
@router.post("/import")
async def import_memories_from_file(
request: Request,
session: str | None = Form(None),
file: UploadFile = File(...)
):
"""Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
from src.auth_helpers import require_privilege
require_privilege(request, "can_manage_memory")
endpoint_url = None
model = None
headers = {}
if session:
try:
sess = session_manager.get_session(session)
_assert_session_owner(sess, _owner(request))
endpoint_url, model, headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
)
except KeyError:
logger.warning("Session %s not found, falling back to utility endpoint", session)
endpoint_url, model, headers = resolve_endpoint("utility", owner=_owner(request))
else:
endpoint_url, model, headers = resolve_task_endpoint(owner=_owner(request))
if not endpoint_url or not model:
raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
filename = file.filename or "upload"
_, ext = os.path.splitext(filename.lower())
allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
if ext not in allowed:
raise HTTPException(400, f"Unsupported file type: {ext}")
# Extract text based on file type
if ext == ".pdf":
from src.document_processor import _process_pdf
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
text = _process_pdf(tmp_path, owner=_owner(request))
finally:
os.unlink(tmp_path)
else:
try:
text = content.decode("utf-8")
except UnicodeDecodeError:
from charset_normalizer import detect
encoding = (detect(content) or {}).get("encoding") or "utf-8"
text = content.decode(encoding, errors="replace")
if not text.strip():
return {"suggestions": [], "message": "No readable content found"}
# Fast path: a .json upload that already looks like a memories export
# (list of {text, category, ...} dicts, or list of strings) round-trips
# directly without spending an LLM call to re-extract its own output.
# Without this, re-importing a memories.json from another account
# ran the file through the extractor, which often re-emitted the
# entries as a numbered list (and the numbering leaked into the
# `text` field).
if ext == ".json":
try:
parsed = json.loads(text)
except json.JSONDecodeError:
parsed = None
if isinstance(parsed, list) and parsed:
direct = []
for item in parsed:
if isinstance(item, dict) and item.get("text"):
direct.append({
"text": _strip_list_prefix(str(item["text"])),
"category": item.get("category") or "fact",
})
elif isinstance(item, str) and item.strip():
direct.append({
"text": _strip_list_prefix(item.strip()),
"category": "fact",
})
if direct:
return {"suggestions": direct, "filename": filename}
# Truncate very long documents
if len(text) > 15000:
text = text[:15000] + "\n[Truncated]"
# Send to LLM for memory extraction
import_prompt = (
"You are a memory extraction assistant. The user uploaded a document. "
"Analyze the text below and extract specific, useful facts — things like "
"names, preferences, jobs, locations, relationships, opinions, projects, "
"goals, contacts, or any other personal details worth remembering.\n\n"
"Rules:\n"
"- Each fact should be a short, self-contained statement\n"
"- Do NOT extract generic knowledge\n"
"- Focus on personal, memorable information\n"
"- If there are no useful facts, return an empty array\n\n"
"Return a JSON array of objects with 'text' and 'category' fields.\n"
"Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
"Return ONLY valid JSON, no markdown fences."
)
try:
raw = await llm_call_async(
endpoint_url,
model,
[
{"role": "system", "content": import_prompt},
{"role": "user", "content": f"Document: {filename}\n\n{text}"},
],
temperature=0.2,
max_tokens=2000,
headers=headers,
)
# Parse JSON
raw = raw.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
suggestions = json.loads(raw)
if isinstance(suggestions, list):
normalized = []
for s in suggestions:
if not s:
continue
if isinstance(s, dict):
s = dict(s)
if s.get("text"):
s["text"] = _strip_list_prefix(str(s["text"]))
normalized.append(s)
else:
normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
suggestions = normalized
else:
suggestions = []
return {"suggestions": suggestions, "filename": filename}
except json.JSONDecodeError:
# Fallback: split by lines, stripping any "1.", "2)" markdown-list
# numbering the model added so saved memories don't keep the prefix.
lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
except Exception as e:
logger.error(f"Memory import extraction failed: {e}")
raise HTTPException(502, f"LLM extraction failed: {str(e)}")
@router.post("/{memory_id}/pin")
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
"""Pin or unpin a memory. Pinned memories are always included in context."""
user = _owner(request)
all_mem = memory_manager.load_all()
for i, memory in enumerate(all_mem):
if memory["id"] == memory_id:
_verify_memory_owner(memory, user)
all_mem[i]["pinned"] = pinned
memory_manager.save(all_mem)
return {"ok": True, "pinned": pinned}
raise HTTPException(404, f"Memory item {memory_id} not found")
# Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
@router.get("/{memory_id}")
def get_memory_item(request: Request, memory_id: str):
"""Get a specific memory item by ID."""
user = _owner(request)
memories = memory_manager.load(owner=user)
for memory in memories:
if memory["id"] == memory_id:
return {"memory": memory}
raise HTTPException(404, "Memory not found")
@router.put("/{memory_id}")
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
"""Update an existing memory item with new text and optional category."""
user = _owner(request)
all_mem = memory_manager.load_all()
for i, memory in enumerate(all_mem):
if memory["id"] == memory_id:
_verify_memory_owner(memory, user)
all_mem[i]["text"] = text.strip()
if category:
all_mem[i]["category"] = category
all_mem[i]["timestamp"] = int(time.time())
memory_manager.save(all_mem)
# Sync vector index (remove old, add updated)
if memory_vector and memory_vector.healthy:
memory_vector.remove(memory_id)
memory_vector.add(memory_id, text.strip())
return {"ok": True, "message": "Memory updated successfully"}
raise HTTPException(404, f"Memory item {memory_id} not found")
@router.delete("/{memory_id}")
def delete_memory(request: Request, memory_id: str):
"""Delete a memory item by its ID."""
user = _owner(request)
all_mem = memory_manager.load_all()
# Find and verify ownership before deleting
target = next((m for m in all_mem if m["id"] == memory_id), None)
if not target:
raise HTTPException(404, f"Memory item {memory_id} not found")
_verify_memory_owner(target, user)
all_mem = [m for m in all_mem if m["id"] != memory_id]
memory_manager.save(all_mem)
# Sync vector index
if memory_vector and memory_vector.healthy:
memory_vector.remove(memory_id)
return {"ok": True, "message": "Memory deleted successfully"}
return router
routes/model_routes.py
+119 -347
View File
@@ -5,7 +5,6 @@ import re
import uuid
import json
import hashlib
import ipaddress
import socket
import time as _time
import logging
@@ -17,9 +16,7 @@ from fastapi import APIRouter, HTTPException, Form, Query, Body, Request, Respon
from pydantic import BaseModel
from fastapi.responses import StreamingResponse
from core.database import SessionLocal, ModelEndpoint, Session as DbSession
from core.log_safety import redact_url as _redact_url_for_log
from core.middleware import require_admin
from src.constants import COOKBOOK_STATE_FILE
from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
from src.tls_overrides import llm_verify
from src.settings import load_settings as _load_settings, save_settings as _save_settings
@@ -113,67 +110,6 @@ def _clear_endpoint_settings_for_endpoint(settings: dict, ep_id: str, *, include
return cleared
_COOKBOOK_ACTIVE_SERVE_STATUSES = {
"starting", "loading", "ready", "running", "restarting",
}
def _active_cookbook_endpoint_ids() -> set[str]:
"""Endpoint IDs owned by active Cookbook serve tasks.
Cookbook auto-registers endpoints with ids like ``local-*``. Those rows are
managed lifecycle state, not durable user configuration. If a tmux stream is
stopped or an old task lingers, the row must stop participating in model
selection and defaults.
"""
try:
if not os.path.exists(COOKBOOK_STATE_FILE):
return set()
with open(COOKBOOK_STATE_FILE, "r", encoding="utf-8") as fh:
raw = fh.read()
state = json.loads(raw)
except Exception:
return set()
out: set[str] = set()
for task in state.get("tasks") or []:
if not isinstance(task, dict) or task.get("type") != "serve":
continue
if str(task.get("status") or "").lower() not in _COOKBOOK_ACTIVE_SERVE_STATUSES:
continue
ep_id = task.get("_endpointId") or task.get("endpointId") or task.get("endpoint_id")
if ep_id:
out.add(str(ep_id))
return out
def _disable_stale_cookbook_local_endpoints(db) -> int:
"""Disable enabled cookbook endpoints whose serve task is no longer active."""
active_ids = _active_cookbook_endpoint_ids()
if not active_ids:
return 0
stale = (
db.query(ModelEndpoint)
.filter(ModelEndpoint.is_enabled == True) # noqa: E712
.filter(ModelEndpoint.id.like("local-%"))
.filter(~ModelEndpoint.id.in_(active_ids))
.all()
)
if not stale:
return 0
settings = _load_settings()
touched_settings = False
for ep in stale:
ep.is_enabled = False
ep.model_refresh_mode = "disabled"
if _clear_endpoint_settings_for_endpoint(settings, ep.id):
touched_settings = True
logger.info("Disabled stale Cookbook endpoint %s (%s @ %s)", ep.id, ep.name, ep.base_url)
if touched_settings:
_save_settings(settings)
db.commit()
return len(stale)
def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
"""Remove endpoint references from scoped or legacy-flat user preferences."""
if not isinstance(all_prefs, dict):
@@ -187,24 +123,7 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
return cleared_users
def _endpoint_visible_model_ids(ep: Any) -> List[str]:
"""Known visible model ids for an endpoint, including pinned/manual ids."""
if ep is None:
return []
return _visible_models(
getattr(ep, "cached_models", None),
getattr(ep, "hidden_models", None),
getattr(ep, "pinned_models", None),
)
def _default_endpoint_needs_assignment(
current_default_id: str,
enabled_endpoint_ids,
*,
current_default_endpoint: Any = None,
current_default_model: str = "",
) -> bool:
def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool:
"""Whether the global default chat endpoint should be (re)assigned.
True when nothing is configured yet, or the configured default no longer
@@ -216,14 +135,7 @@ def _default_endpoint_needs_assignment(
"""
if not current_default_id:
return True
if current_default_id not in enabled_endpoint_ids:
return True
if current_default_endpoint is None:
return False
if not (current_default_model or "").strip():
return True
visible = _endpoint_visible_model_ids(current_default_endpoint)
return bool(visible and current_default_model not in visible)
return current_default_id not in enabled_endpoint_ids
# Loopback hosts a user might type for a local model server (LM Studio,
@@ -493,11 +405,8 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float:
except Exception:
val = 0
if val > 0:
return float(max(1, min(60, val)))
# llama.cpp and other local OpenAI-compatible servers can block briefly
# while warming/loading. A 2s local timeout makes working endpoints flicker
# offline before /v1/models is ready.
return 10.0 if category == "local" else 2.0
return float(max(1, min(30, val)))
return 2.5 if category == "local" else 2.0
def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float:
@@ -564,7 +473,7 @@ def _explicit_model_list_timeout(base_url: str, endpoint_kind: str = "auto", req
category = _classify_endpoint(base_url, kind)
if kind in ("api", "proxy") or category == "api":
return 30.0
return 15.0 if category == "local" else (3.0 if _is_ollama_base(base_url) else 2.0)
return 3.0 if _is_ollama_base(base_url) else 2.0
def _cached_model_ids(ep: Any) -> List[str]:
@@ -609,10 +518,6 @@ _NON_CHAT_EXACT_PREFIXES = (
def _is_chat_model(model_id: str) -> bool:
"""Return True if the model ID looks like a chat/completions-capable model."""
if not isinstance(model_id, str):
# Non-compliant upstreams can return non-string IDs (e.g. int/None);
# treat them as chat-capable rather than crashing on .lower().
return True
mid = model_id.lower()
for prefix in _NON_CHAT_PREFIXES:
if mid.startswith(prefix):
@@ -657,8 +562,6 @@ def _safe_build_models_url(base_url: str) -> str:
"""Build a /models URL without letting optional provider imports break probes."""
try:
return build_models_url(base_url)
except ValueError:
raise
except Exception as exc:
logger.debug("Model URL detection failed for %s: %s", base_url, exc)
return f"{(base_url or '').rstrip('/')}/models"
@@ -730,7 +633,7 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1
try:
t0 = _time.time()
r = httpx.post(target_url, headers=h, json=payload, timeout=timeout, verify=llm_verify())
r = httpx.post(target_url, headers=h, json=payload, timeout=timeout)
latency = round((_time.time() - t0) * 1000)
if r.is_success:
return {"status": "ok", "latency_ms": latency}
@@ -756,20 +659,13 @@ def _probe_single_model(base: str, api_key: str, model_id: str, timeout: int = 1
# Hostnames / IP prefixes that indicate a local endpoint
_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"}
_PRIVATE_NETWORKS = (
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.168.0.0/16"),
)
_TAILSCALE_CGNAT = ipaddress.ip_network("100.64.0.0/10")
_PRIVATE_PREFIXES = ("10.", "172.16.", "172.17.", "172.18.", "172.19.",
"172.20.", "172.21.", "172.22.", "172.23.", "172.24.",
"172.25.", "172.26.", "172.27.", "172.28.", "172.29.",
"172.30.", "172.31.", "192.168.")
def _local_ip_literal(host: str) -> bool:
try:
ip = ipaddress.ip_address(host)
except ValueError:
return False
return any(ip in network for network in _PRIVATE_NETWORKS) or ip in _TAILSCALE_CGNAT
_TAILSCALE_RE = re.compile(r"^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.")
def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str:
@@ -783,7 +679,9 @@ def _classify_endpoint(base_url: str, endpoint_kind: str = "auto") -> str:
return "api"
try:
host = urlparse(base_url).hostname or ""
if host in _LOCAL_HOSTS or _local_ip_literal(host):
if host in _LOCAL_HOSTS or host.startswith(_PRIVATE_PREFIXES):
return "local"
if _TAILSCALE_RE.match(host):
return "local"
except Exception:
pass
@@ -805,51 +703,6 @@ def _effective_endpoint_kind(ep: Any, base_url: str) -> str:
return "auto"
def _is_loading_model_response(resp: Any) -> bool:
if getattr(resp, "status_code", None) != 503:
return False
try:
body = resp.text or ""
except Exception:
body = ""
return "loading model" in body.lower()
def _openai_model_ids(data: Any) -> List[str]:
"""Extract OpenAI-style model IDs.
Accepts both standard ``{"data": [{"id": ...}]}`` responses and bare
``[{"id": ...}]`` lists returned by some OpenAI-compatible providers.
Tolerates non-dict/non-list bodies and non-string IDs, returning only
non-empty string IDs.
"""
if isinstance(data, list):
items = data
elif isinstance(data, dict):
items = data.get("data")
else:
items = None
return [m["id"] for m in (items or [])
if isinstance(m, dict) and isinstance(m.get("id"), str) and m["id"]]
def _ollama_model_names(data: Any) -> List[str]:
"""Extract native-Ollama model names (``{"models": [{"name"|"model": ...}]}``).
Same tolerance as :func:`_openai_model_ids`: a non-dict body or non-string
value is skipped rather than crashing, preserving name-then-model precedence.
"""
items = data.get("models") if isinstance(data, dict) else None
out: List[str] = []
for m in (items or []):
if not isinstance(m, dict):
continue
v = m.get("name") or m.get("model")
if isinstance(v, str) and v:
out.append(v)
return out
def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]:
"""Probe a base URL's /models endpoint and return list of model IDs.
@@ -873,7 +726,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
r = httpx.get(url, headers=headers, timeout=timeout, verify=llm_verify())
r.raise_for_status()
data = r.json()
models = _openai_model_ids(data)
models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
if models:
return models
except httpx.HTTPStatusError as e:
@@ -895,10 +748,10 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
r.raise_for_status()
data = r.json()
# OpenAI format: {"data": [{"id": "model-name"}]}
models = _openai_model_ids(data)
models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
# Ollama format: {"models": [{"name": "model-name"}]}
if not models:
models = _ollama_model_names(data)
models = [m.get("name") or m.get("model") for m in (data.get("models") or []) if m.get("name") or m.get("model")]
if models:
# Z.AI coding plan omits some working models from /models;
# append curated-only entries for that endpoint only.
@@ -914,19 +767,16 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
models.append(_e)
return [m for m in models if _is_chat_model(m)]
except httpx.HTTPStatusError as e:
if e.response is not None and _is_loading_model_response(e.response):
logger.info("Endpoint still loading model at %s", _redact_url_for_log(url))
return []
if api_key:
status = e.response.status_code if e.response is not None else "unknown"
logger.warning("Failed to probe %s with API key: HTTP %s", _redact_url_for_log(url), status)
logger.warning(f"Failed to probe {url} with API key: HTTP {status}")
return []
logger.warning("Failed to probe %s: %s", _redact_url_for_log(url), e)
logger.warning(f"Failed to probe {url}: {e}")
except Exception as e:
if api_key:
logger.warning("Failed to probe %s with API key: %s", _redact_url_for_log(url), e)
logger.warning(f"Failed to probe {url} with API key: {e}")
return []
logger.warning("Failed to probe %s: %s", _redact_url_for_log(url), e)
logger.warning(f"Failed to probe {url}: {e}")
# Older Ollama builds and some proxies expose native /api/tags even when
# the OpenAI-compatible /v1/models path is unavailable.
@@ -937,7 +787,7 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
r = httpx.get(root + "/api/tags", timeout=timeout, verify=llm_verify())
r.raise_for_status()
data = r.json()
models = _ollama_model_names(data)
models = [m.get("name") or m.get("model") for m in (data.get("models") or []) if m.get("name") or m.get("model")]
if models:
return [m for m in models if _is_chat_model(m)]
except Exception as e:
@@ -966,15 +816,6 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
or "ollama" in (parsed_base.hostname or "").lower()
)
def _is_loading_model_response(r) -> bool:
if getattr(r, "status_code", None) != 503:
return False
try:
body = r.text or ""
except Exception:
body = ""
return "loading model" in body.lower()
def _result_from_response(r) -> Dict[str, Any]:
if 300 <= r.status_code < 400:
loc = r.headers.get("location", "")
@@ -991,13 +832,6 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
"status_code": r.status_code,
"error": None,
}
if _is_loading_model_response(r):
return {
"reachable": True,
"loading": True,
"status_code": r.status_code,
"error": "Loading model",
}
return {"reachable": False, "status_code": r.status_code, "error": f"HTTP {r.status_code}"}
last_error: Optional[str] = None
@@ -1030,7 +864,7 @@ def _ping_endpoint(base_url: str, api_key: str = None, timeout: float = 1.5) ->
if 400 <= sc < 500 and sc not in (401, 403):
models_url = _safe_build_models_url(base)
try:
r2 = httpx.get(models_url, headers=headers,timeout=timeout, verify=llm_verify())
r2 = httpx.get(models_url, headers=headers, timeout=timeout, verify=llm_verify())
result2 = _result_from_response(r2)
if result2["reachable"]:
return result2
@@ -1152,36 +986,6 @@ def _merge_model_ids(*lists):
return out
def _is_mlx_deepseek_v4_repo_id(model_id: str) -> bool:
m = str(model_id or "").lower()
return "mlx-community/deepseek-v4" in m
def _is_mlx_deepseek_v4_shim_id(model_id: str) -> bool:
m = str(model_id or "").lower()
return "/.cache/odysseus/mlx-shims/deepseek-v4" in m
def _filter_mlx_deepseek_v4_repo_when_shimmed(model_ids):
"""Hide the broken MLX repo id when a launch-specific shim id is available.
mlx_lm.server may advertise the original HF repo id even though generation
only works through Odysseus' sanitized local shim. Keep the shim as the
submitted model id and remove the raw repo id from the picker/default list.
"""
ids = list(model_ids or [])
has_shim = any(_is_mlx_deepseek_v4_shim_id(m) for m in ids)
if not has_shim:
return ids
return [m for m in ids if not _is_mlx_deepseek_v4_repo_id(m)]
def _model_display_name(model_id: str) -> str:
if _is_mlx_deepseek_v4_shim_id(model_id):
return str(model_id or "").rstrip("/").split("/")[-1] or "DeepSeek-V4-Flash-4bit"
return str(model_id or "").split("/")[-1]
def _visible_models(cached_models, hidden_models, pinned_models=None):
"""Merge cached + pinned model IDs, then filter out hidden ones.
@@ -1195,7 +999,6 @@ def _visible_models(cached_models, hidden_models, pinned_models=None):
_normalize_model_ids(cached_models),
_normalize_model_ids(pinned_models),
)
merged = _filter_mlx_deepseek_v4_repo_when_shimmed(merged)
if not hidden_models:
return merged
hidden = set(_normalize_model_ids(hidden_models))
@@ -1245,11 +1048,9 @@ def setup_model_routes(model_discovery):
except Exception:
return 0.0
def _failure_delay(fails: int, *, empty_local: bool = False) -> float:
def _failure_delay(fails: int) -> float:
if fails <= 0:
return 0.0
if empty_local:
return min(5.0 * (2 ** max(0, fails - 1)), 30.0)
return min(_REFRESH_FAILURE_BASE * (2 ** max(0, fails - 1)), _REFRESH_FAILURE_MAX)
def _should_refresh_endpoint(ep: Any, now: float, force: bool = False) -> tuple[bool, Dict[str, Any]]:
@@ -1280,12 +1081,7 @@ def setup_model_routes(model_discovery):
fails = int(state.get("fail_count") or 0)
if fails and not force:
last_failure = float(state.get("last_failure") or 0.0)
empty_local = (
not cached
and category == "local"
and str(getattr(ep, "id", "") or "").startswith("local-")
)
if now - last_failure < _failure_delay(fails, empty_local=empty_local):
if now - last_failure < _failure_delay(fails):
return False, info
if cached and not force:
interval = _endpoint_refresh_interval(ep, category)
@@ -1311,8 +1107,6 @@ def setup_model_routes(model_discovery):
db = SessionLocal()
changed = False
try:
if _disable_stale_cookbook_local_endpoints(db):
changed = True
endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
now = _time.time()
groups: Dict[str, Dict[str, Any]] = {}
@@ -1386,8 +1180,6 @@ def setup_model_routes(model_discovery):
db = SessionLocal()
try:
if _disable_stale_cookbook_local_endpoints(db):
_invalidate_models_cache()
q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
if owner and not is_admin:
# Regular users see: their own endpoints + null-owner
@@ -1427,9 +1219,9 @@ def setup_model_routes(model_discovery):
"port": 0,
"url": chat_url,
"models": curated,
"models_display": [_model_display_name(mid) for mid in curated],
"models_display": [mid.split("/")[-1] for mid in curated],
"models_extra": extra,
"models_extra_display": [_model_display_name(mid) for mid in extra],
"models_extra_display": [mid.split("/")[-1] for mid in extra],
"endpoint_id": ep.id,
"endpoint_name": ep.name,
"category": category,
@@ -1457,7 +1249,7 @@ def setup_model_routes(model_discovery):
return {"hosts": [], "items": items}
@router.get("/models")
def api_models(request: Request, refresh: bool = False, background: bool = False):
def api_models(request: Request, refresh: bool = False):
"""Get available models — per-user (caller sees only their endpoints +
legacy/shared null-owner rows). Cached per-user for 30s."""
# Require auth; "" is the unconfigured single-user mode, treated as
@@ -1499,11 +1291,8 @@ def setup_model_routes(model_discovery):
return cache_entry["data"]
result = _fetch_models(owner=owner, is_admin=_is_admin)
_models_cache[_cache_key] = {"data": result, "time": now}
# Kick off background refresh to update caches from live endpoints.
# Page boot can opt out with background=false so opening Odysseus does
# not start endpoint probes against slow/offline model servers.
if background or refresh:
_refresh_caches_bg(force=refresh)
# Kick off background refresh to update caches from live endpoints
_refresh_caches_bg(force=refresh)
return result
# Brief cache for local-probe results so picker-open doesn't hammer
@@ -1512,7 +1301,6 @@ def setup_model_routes(model_discovery):
# within ~8s of the user noticing.
_LOCAL_PROBE_TTL = 8.0
_local_probe_cache: Dict[str, Any] = {"data": None, "time": 0.0}
_local_probe_inflight: Dict[str, Any] = {"task": None}
@router.get("/model-endpoints/probe-local")
async def probe_local_endpoints(request: Request):
@@ -1527,72 +1315,58 @@ def setup_model_routes(model_discovery):
(now - _local_probe_cache["time"]) < _LOCAL_PROBE_TTL):
return _local_probe_cache["data"]
import asyncio as _asyncio
task = _local_probe_inflight.get("task")
if task is not None and not task.done():
return await task
async def _compute_local_probe() -> Dict[str, Any]:
db = SessionLocal()
try:
if _disable_stale_cookbook_local_endpoints(db):
_invalidate_models_cache()
endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
local_eps = []
for ep in endpoints:
base = _normalize_base(ep.base_url)
kind = _effective_endpoint_kind(ep, base)
if _classify_endpoint(base, kind) == "local":
local_eps.append((ep.id, base, ep.api_key))
finally:
db.close()
grouped: Dict[str, Dict[str, Any]] = {}
for ep_id, base, api_key in local_eps:
key = _refresh_key(base, api_key)
grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
t0 = _time.time()
try:
# Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
# local vLLM endpoints on Tailscale links where the model
# server is still loading (Qwen3.5-122B takes 2–3 min to
# warm); /v1/models can take 500–2500 ms on a busy box,
# which pushed _ping_endpoint's full path-discovery sweep
# past the cap and marked the row offline despite the
# user actively chatting with it.
ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
lat = round((_time.time() - t0) * 1000)
return {
"alive": bool(ping.get("reachable")),
"latency_ms": lat,
"status_code": ping.get("status_code"),
"error": ping.get("error"),
}
except Exception as e:
return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
results_list = await _asyncio.gather(
*[_probe_one(data) for data in grouped.values()],
return_exceptions=False,
)
results: Dict[str, Any] = {}
for data, r in zip(grouped.values(), results_list):
for eid in data["endpoint_ids"]:
results[eid] = r
_local_probe_cache["data"] = results
_local_probe_cache["time"] = _time.time()
return results
task = _asyncio.create_task(_compute_local_probe())
_local_probe_inflight["task"] = task
db = SessionLocal()
try:
return await task
endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
local_eps = []
for ep in endpoints:
base = _normalize_base(ep.base_url)
kind = _effective_endpoint_kind(ep, base)
if _classify_endpoint(base, kind) == "local":
local_eps.append((ep.id, base, ep.api_key))
finally:
if _local_probe_inflight.get("task") is task:
_local_probe_inflight["task"] = None
db.close()
grouped: Dict[str, Dict[str, Any]] = {}
for ep_id, base, api_key in local_eps:
key = _refresh_key(base, api_key)
grouped.setdefault(key, {"base": base, "api_key": api_key, "endpoint_ids": []})["endpoint_ids"].append(ep_id)
async def _probe_one(data: Dict[str, Any]) -> Dict[str, Any]:
t0 = _time.time()
try:
import asyncio as _asyncio
# Bumped 1.5s → 3.5s. The previous 1.5s budget was clipping
# local vLLM endpoints on Tailscale links where the model
# server is still loading (Qwen3.5-122B takes 2–3 min to
# warm); /v1/models can take 500–2500 ms on a busy box,
# which pushed _ping_endpoint's full path-discovery sweep
# past the cap and marked the row offline despite the
# user actively chatting with it.
ping = await _asyncio.to_thread(_ping_endpoint, data["base"], data.get("api_key"), 3.5)
lat = round((_time.time() - t0) * 1000)
return {
"alive": bool(ping.get("reachable")),
"latency_ms": lat,
"status_code": ping.get("status_code"),
"error": ping.get("error"),
}
except Exception as e:
return {"alive": False, "latency_ms": None, "status_code": None, "error": str(e)[:120]}
import asyncio as _asyncio
results_list = await _asyncio.gather(
*[_probe_one(data) for data in grouped.values()],
return_exceptions=False,
)
results: Dict[str, Any] = {}
for data, r in zip(grouped.values(), results_list):
for eid in data["endpoint_ids"]:
results[eid] = r
_local_probe_cache["data"] = results
_local_probe_cache["time"] = now
return results
@router.get("/ping")
def ping_endpoints(request: Request):
@@ -1622,7 +1396,7 @@ def setup_model_routes(model_discovery):
t0 = _time.time()
ping = _ping_endpoint(base, ep.api_key, timeout=1.5)
entry["latency_ms"] = round((_time.time() - t0) * 1000)
entry["status"] = "loading" if ping.get("loading") else ("online" if ping.get("reachable") or cached_count else "offline")
entry["status"] = "online" if ping.get("reachable") or cached_count else "offline"
entry["error"] = ping.get("error")
entry["model_count"] = cached_count or (len(ANTHROPIC_MODELS) if provider == "anthropic" else 0)
except Exception as e:
@@ -1775,8 +1549,6 @@ def setup_model_routes(model_discovery):
require_admin(request)
db = SessionLocal()
try:
if _disable_stale_cookbook_local_endpoints(db):
_invalidate_models_cache()
rows = db.query(ModelEndpoint).order_by(ModelEndpoint.created_at).all()
results = []
for r in rows:
@@ -1784,11 +1556,39 @@ def setup_model_routes(model_discovery):
hidden = _hidden_model_ids(r)
pinned = _normalize_model_ids(getattr(r, "pinned_models", None))
visible = _visible_models(all_models, r.hidden_models, pinned)
# Keep the list route cache-only. It feeds Settings →
# Added Models and must render immediately; explicit
# Refresh/Probe endpoints do the network work.
status = "online" if (all_models or pinned) else ("empty" if r.is_enabled else "offline")
# Endpoint counts as reachable if it has any model — including
# admin-pinned IDs that a probe would never surface.
status = "online" if (all_models or pinned) else "offline"
ping = None
# When cached_models is empty, do a quick reachability probe.
# Bumped 1.0s → 3.5s because the user reported endpoints they
# were ACTIVELY chatting with showed "offline" — the previous
# 1s timeout was clipping live cloud endpoints (DeepSeek can
# take 1.5–2.5s on /v1/models when their region is under load,
# vLLM on a remote GPU box behind SSH can also push past 1s).
# 3.5s still keeps the picker render snappy in the common
# "everything's already cached" path because this branch only
# runs for endpoints with an empty cached_models.
if not all_models and not pinned and r.is_enabled:
ping = _ping_endpoint(r.base_url, r.api_key, timeout=3.5)
if ping.get("reachable"):
status = "empty"
# Best-effort: if the probe came back reachable, try
# to populate cached_models in the background so the
# NEXT picker load shows "online" instead of "empty".
# Failure here is silent — we already returned the
# "empty" status, and the existing background refresh
# path will eventually fill it in too.
try:
probed = _probe_endpoint(r.base_url, r.api_key, timeout=5)
if probed:
r.cached_models = json.dumps(probed)
db.commit()
all_models = probed
visible = _visible_models(all_models, r.hidden_models, pinned)
status = "online"
except Exception as _refill_err:
logger.debug(f"opportunistic cached_models refill failed for {r.id}: {_refill_err!r}")
base = _normalize_base(r.base_url)
kind = _effective_endpoint_kind(r, base)
results.append({
@@ -1918,14 +1718,7 @@ def setup_model_routes(model_discovery):
if api_key.strip() and not existing.api_key:
existing.api_key = api_key.strip()
changed = True
# Keep duplicate endpoint registration cheap. This path is hit
# by Cookbook/browser auto-register flows and can run while the
# user is sending a chat message. Probing a stale LAN endpoint
# here used to hold the request open for tens of seconds and
# contend with session creation, making "send" feel blocked.
# Explicit "require models" calls still probe; normal refresh
# belongs to /model-endpoints/{id}/models or /probe.
if require_model_list:
if should_probe:
probed_models = _probe_endpoint(
base_url,
(api_key.strip() or existing.api_key or None),
@@ -1965,7 +1758,7 @@ def setup_model_routes(model_discovery):
model_ids = _probe_endpoint(base_url, api_key.strip() or None, timeout=explicit_timeout) if should_probe else []
ping = {"reachable": False, "error": None}
if (should_probe or requested_kind in ("api", "proxy")) and not model_ids:
ping = _ping_endpoint(base_url, api_key.strip() or None, timeout=min(explicit_timeout, 10.0))
ping = _ping_endpoint(base_url, api_key.strip() or None, timeout=min(explicit_timeout, 2.0))
if require_model_list and not model_ids:
raise HTTPException(400, _model_endpoint_error_message(base_url, ping))
@@ -2012,18 +1805,7 @@ def setup_model_routes(model_discovery):
ModelEndpoint.is_enabled == True # noqa: E712
).all()
}
current_default_id = settings.get("default_endpoint_id") or ""
current_default_ep = None
if current_default_id:
current_default_ep = db.query(ModelEndpoint).filter(
ModelEndpoint.id == current_default_id
).first()
if _default_endpoint_needs_assignment(
current_default_id,
enabled_ids,
current_default_endpoint=current_default_ep,
current_default_model=settings.get("default_model") or "",
):
if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids):
from src.endpoint_resolver import _first_chat_model
settings["default_endpoint_id"] = ep.id
settings["default_model"] = _first_chat_model(model_ids) or ""
@@ -2043,7 +1825,7 @@ def setup_model_routes(model_discovery):
"models": _merge_model_ids(model_ids, _pinned),
"pinned_models": _pinned,
"online": bool(model_ids) or bool(_pinned) or bool(ping.get("reachable")),
"status": "online" if (model_ids or _pinned) else ("loading" if ping.get("loading") else ("empty" if ping.get("reachable") else "offline")),
"status": "online" if (model_ids or _pinned) else ("empty" if ping.get("reachable") else "offline"),
"ping_error": ping.get("error") if ping else None,
"endpoint_kind": requested_kind,
"category": _classify_endpoint(base_url, requested_kind),
@@ -2068,11 +1850,11 @@ def setup_model_routes(model_discovery):
configured_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60)
probe_timeout = _explicit_model_list_timeout(base_url, requested_kind, configured_timeout)
models = _probe_endpoint(base_url, api_key.strip() or None, timeout=probe_timeout)
ping = {"reachable": True, "error": None} if models else _ping_endpoint(base_url, api_key.strip() or None, timeout=min(probe_timeout, 10.0))
ping = {"reachable": True, "error": None} if models else _ping_endpoint(base_url, api_key.strip() or None, timeout=min(probe_timeout, 2.0))
return {
"base_url": base_url,
"online": bool(models) or bool(ping.get("reachable")),
"status": "online" if models else ("loading" if ping.get("loading") else ("empty" if ping.get("reachable") else "offline")),
"status": "online" if models else ("empty" if ping.get("reachable") else "offline"),
"ping_error": ping.get("error") if ping else None,
"models": models,
"count": len(models),
@@ -2250,16 +2032,6 @@ def setup_model_routes(model_discovery):
ep_id = (_user_prefs.get("default_endpoint_id") or "").strip()
model = (_user_prefs.get("default_model") or "").strip()
_fallbacks = _user_prefs.get("default_model_fallbacks") or []
# If user has no personal default, fall back to global default
# But only based on the "share_defaults_with_users" flag
# (only if share_defaults_with_users is enabled)
if settings.get("share_defaults_with_users", False):
if not ep_id:
ep_id = settings.get("default_endpoint_id", "")
if not model:
model = settings.get("default_model", "")
if not _fallbacks:
_fallbacks = settings.get("default_model_fallbacks") or []
else:
ep_id = settings.get("default_endpoint_id", "")
model = settings.get("default_model", "")
routes/note_routes.py
+12 -47
View File
@@ -10,10 +10,8 @@ from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel
from core.database import SessionLocal, Note
from core.middleware import INTERNAL_TOOL_USER
from src.auth_helpers import require_user
from src.constants import DATA_DIR
from src.upload_handler import reserve_upload_references
from sqlalchemy.orm.attributes import flag_modified
logger = logging.getLogger(__name__)
@@ -336,11 +334,10 @@ async def dispatch_reminder(
# Loud diagnostic so we can see WHY a reminder didn't send (the
# previous "silently no-op when cfg has no smtp_host" was invisible).
logger.info(
"dispatch_reminder[email] note_id=%s owner=%r "
"has_smtp_host=%s has_smtp_user=%s has_from=%s has_recipient=%s",
note_id, owner,
bool(cfg.get("smtp_host")), bool(cfg.get("smtp_user")),
bool(from_addr), bool(recipient),
f"dispatch_reminder[email] note_id={note_id} owner={owner!r} "
f"smtp_host={cfg.get('smtp_host')!r} smtp_user={cfg.get('smtp_user')!r} "
f"from={from_addr!r} recipient={recipient!r} "
f"account_name={cfg.get('account_name')!r}"
)
missing = []
if not cfg.get("smtp_host"):
@@ -480,28 +477,15 @@ async def dispatch_reminder(
base = intg["base_url"].rstrip("/")
topic = settings.get("reminder_ntfy_topic") or "reminders"
ntfy_body = synthesis or note_body or title
# ntfy Title is an ASCII HTTP header; sanitize Unicode and cap its length.
_clean_title = (title or "Reminder").encode("ascii", "replace").decode("ascii")[:200]
hdrs = {"Title": _clean_title, "Priority": "high", "Tags": "bell"}
hdrs = {"Title": title or "Reminder", "Priority": "high", "Tags": "bell"}
api_key = intg.get("api_key", "")
if api_key:
hdrs["Authorization"] = f"Bearer {api_key}"
# SSRF guard — same check (and env knob) as the webhook branch
# above: link-local / metadata addresses are always rejected;
# REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
# so a ntfy base_url can't be pointed at internal services.
import os as _os
from src.url_safety import check_outbound_url as _chk
_block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
_ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
if not _ok:
ntfy_error = f"ntfy URL rejected: {_reason}"
else:
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
ntfy_sent = resp.is_success
if not ntfy_sent:
ntfy_error = f"ntfy returned HTTP {resp.status_code}"
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
ntfy_sent = resp.is_success
if not ntfy_sent:
ntfy_error = f"ntfy returned HTTP {resp.status_code}"
else:
ntfy_error = "No enabled ntfy integration"
except Exception as e:
@@ -575,7 +559,7 @@ async def dispatch_reminder(
# Router factory
# ---------------------------------------------------------------------------
def setup_note_routes(task_scheduler=None, upload_handler=None):
def setup_note_routes(task_scheduler=None):
# Expose the scheduler to module-level `dispatch_reminder` so reminders
# can also push to the in-app notification queue (the polling system
# turns each entry into a real browser Notification + the existing
@@ -597,13 +581,8 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
# did not.
return require_user(request) or None
def _reserve_note_uploads(owner: Optional[str], *values) -> None:
missing_id = reserve_upload_references(upload_handler, owner, *values)
if missing_id:
raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
def _is_admin_or_single_user(request: Request, user: str | None) -> bool:
if user == INTERNAL_TOOL_USER:
if user == "internal-tool":
return True
if not user:
# require_user() already admitted this request, which only happens
@@ -651,13 +630,6 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
@router.post("")
def create_note(request: Request, body: NoteCreate):
user = _owner(request)
_reserve_note_uploads(
user,
body.image_url,
body.color,
body.content,
json.dumps(body.items) if body.items is not None else None,
)
db = SessionLocal()
try:
note = Note(
@@ -715,13 +687,6 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
if user is not None and note.owner != user:
raise HTTPException(404, "Note not found")
_reserve_note_uploads(
user,
body.image_url,
body.color,
body.content,
json.dumps(body.items) if body.items is not None else None,
)
if body.title is not None:
note.title = body.title
if body.content is not None:
routes/personal_routes.py
+5 -90
View File
@@ -2,9 +2,8 @@
"""Routes for personal documents management."""
import os
import logging
import shutil
import uuid
from typing import Any, Dict, List, Tuple
from typing import List, Tuple
from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends
from src.request_models import DirectoryRequest
from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR
@@ -19,15 +18,14 @@ UPLOADS_DIR = PERSONAL_UPLOADS_DIR
logger = logging.getLogger(__name__)
def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str:
def _personal_upload_dir_for_owner(owner: str | None) -> str:
"""Return the per-owner upload directory used for direct RAG uploads."""
owner_segment = secure_filename((owner or "local").strip())[:80] or "local"
upload_dir = os.path.abspath(os.path.join(UPLOADS_DIR, owner_segment))
base_abs = os.path.abspath(UPLOADS_DIR)
if os.path.commonpath([upload_dir, base_abs]) != base_abs:
raise ValueError("Unsafe upload owner path")
if create:
os.makedirs(upload_dir, exist_ok=True)
os.makedirs(upload_dir, exist_ok=True)
return upload_dir
@@ -46,87 +44,6 @@ def _unique_personal_upload_path(upload_dir: str, original_name: str | None) ->
raise ValueError("Unsafe upload filename")
return file_path, filename, safe_name
def _unique_existing_target(path: str) -> str:
"""Return a non-existing sibling path for rename collision handling."""
if not os.path.exists(path):
return path
stem, ext = os.path.splitext(path)
while True:
candidate = f"{stem}-{uuid.uuid4().hex[:10]}{ext}"
if not os.path.exists(candidate):
return candidate
def _remove_empty_tree(path: str) -> None:
"""Best-effort removal of empty directories under ``path``."""
if not os.path.isdir(path):
return
for root, dirs, _files in os.walk(path, topdown=False):
for dirname in dirs:
candidate = os.path.join(root, dirname)
try:
os.rmdir(candidate)
except OSError:
pass
try:
os.rmdir(path)
except OSError:
pass
def rename_personal_upload_owner(
old_owner: str,
new_owner: str,
*,
personal_docs_manager: Any = None,
rag_manager: Any = None,
) -> Dict[str, Any]:
"""Move direct personal uploads and rewrite RAG owner metadata on user rename."""
old_dir = _personal_upload_dir_for_owner(old_owner, create=False)
new_dir = _personal_upload_dir_for_owner(new_owner, create=False)
path_map: Dict[str, str] = {}
moved_files = 0
if os.path.isdir(old_dir) and old_dir != new_dir:
os.makedirs(new_dir, exist_ok=True)
for root, _dirs, files in os.walk(old_dir):
rel_root = os.path.relpath(root, old_dir)
target_root = new_dir if rel_root == "." else os.path.join(new_dir, rel_root)
os.makedirs(target_root, exist_ok=True)
for filename in files:
source = os.path.abspath(os.path.join(root, filename))
target = _unique_existing_target(os.path.abspath(os.path.join(target_root, filename)))
shutil.move(source, target)
path_map[source] = target
moved_files += 1
_remove_empty_tree(old_dir)
if personal_docs_manager is not None:
rename_directory = getattr(personal_docs_manager, "rename_directory", None)
if callable(rename_directory):
rename_directory(old_dir, new_dir, path_map=path_map)
rag_result = None
if rag_manager is not None:
rename_owner = getattr(rag_manager, "rename_owner", None)
if callable(rename_owner):
rag_result = rename_owner(
old_owner,
new_owner,
path_map=path_map,
path_prefixes=[(old_dir, new_dir)],
)
return {
"old_dir": old_dir,
"new_dir": new_dir,
"moved_files": moved_files,
"path_map": path_map,
"rag_result": rag_result,
}
def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
"""
Setup personal documents related routes.
@@ -358,13 +275,11 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
except Exception as e:
logger.warning(f"RAG removal failed for {filepath}: {e}")
# Delete file from disk if it's in the caller's own uploads dir.
# Scope to the per-owner subdir, not the shared uploads root, so one
# admin can't delete another user's personal files by path.
# Delete file from disk if it's in uploads dir
deleted_from_disk = False
try:
abs_target = os.path.realpath(filepath)
base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False))
base_abs = os.path.realpath(UPLOADS_DIR)
in_uploads = (
abs_target == base_abs
or os.path.commonpath([abs_target, base_abs]) == base_abs
routes/preset_routes.py
+1 -2
View File
@@ -1,6 +1,5 @@
"""Preset routes — /api/presets GET, /api/presets/custom POST, user templates CRUD."""
import asyncio
import logging
import uuid
from typing import Dict, Any, List
@@ -103,7 +102,7 @@ def setup_preset_routes(preset_manager) -> APIRouter:
try:
model_spec = data.get("model") or ""
user = effective_user(request)
url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=user)
url, model, headers = _resolve_model(model_spec, owner=user)
result = await llm_call_async(url, model, messages, temperature=0.8, max_tokens=500, headers=headers)
return {"success": True, "prompt": result.strip()}
except Exception as e:
routes/research/__init__.py
-5
View File
@@ -1,5 +0,0 @@
"""Research route domain package (slice 2b, #4082/#4071).
Contains research_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/research_routes.py re-exports from here.
"""
routes/research/research_routes.py
-783
View File
@@ -1,783 +0,0 @@
"""Research background task routes — /api/research/*."""
import asyncio
import json
import logging
import re
import uuid
from datetime import datetime
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import HTMLResponse, StreamingResponse
from pydantic import BaseModel, Field
from core.middleware import INTERNAL_TOOL_USER
from src.endpoint_resolver import resolve_endpoint
from src.auth_helpers import _auth_disabled, get_current_user
from core.auth import RESERVED_USERNAMES
from src.constants import DEEP_RESEARCH_DIR
_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
def _validate_session_id(session_id: str) -> str:
if not _SESSION_ID_RE.fullmatch(session_id):
raise HTTPException(400, "Invalid session ID format")
return session_id
def _research_storage_root() -> Path:
return Path(DEEP_RESEARCH_DIR).resolve()
def _find_research_path(session_id: str) -> Path | None:
"""Find a persisted research file without deriving its path from input."""
expected_name = f"{_validate_session_id(session_id)}.json"
root = _research_storage_root()
for stored_path in root.glob("*.json"):
if stored_path.name != expected_name:
continue
resolved = stored_path.resolve()
try:
resolved.relative_to(root)
except ValueError:
return None
if not resolved.is_file():
return None
return resolved
return None
def _require_research_path(session_id: str) -> Path:
path = _find_research_path(session_id)
if path is None:
raise HTTPException(404, "Research not found")
return path
def _find_owned_research_path(session_id: str, user: str) -> Path | None:
path = _find_research_path(session_id)
if path is None:
return None
try:
owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
except Exception:
return None
if owner != user:
return None
return path
logger = logging.getLogger(__name__)
# Model-name substrings that are NOT chat/generation models — research must
# never pick these as its model. An OpenAI-style endpoint often lists
# `text-embedding-ada-002` etc. first in its model list, which is why research
# was failing with "Cannot reach model 'text-embedding-ada-002'".
_NON_CHAT_MODEL = (
"text-embedding", "embedding", "tts-", "whisper", "dall-e",
"moderation", "rerank", "reranker", "clip", "stable-diffusion",
)
_RESEARCH_IMAGE_BLOCKLIST = {
"cdn.shopify.com/s/files/1/0179/4388/7926/files/icon.png",
}
def _is_research_icon_or_logo_url(url: str) -> bool:
path = url.lower().split("?")[0]
return any(token in path for token in (
"/logo", "logo_", "-logo", "favicon", "apple-touch-icon",
"sprite", "icon-", "_icon", "/icons/", "badge",
))
def _research_thumbnail(data: dict) -> str:
"""Pick the same first visible image the visual report uses as hero."""
hidden = set(data.get("hidden_images") or [])
seen = set()
def usable(image: str) -> bool:
image = str(image or "").strip()
if not image or image in seen or image in hidden:
return False
if not image.startswith("https://"):
return False
if image.endswith((".svg", ".ico", ".gif")):
return False
if any(blocked in image for blocked in _RESEARCH_IMAGE_BLOCKLIST):
return False
if _is_research_icon_or_logo_url(image):
return False
return True
for source in data.get("sources") or []:
if not isinstance(source, dict):
continue
image = str(source.get("image") or source.get("og_image") or "").strip()
if usable(image):
seen.add(image)
return image
for finding in data.get("raw_findings") or data.get("findings") or []:
if not isinstance(finding, dict):
continue
image = str(finding.get("image") or finding.get("og_image") or "").strip()
if usable(image):
seen.add(image)
return image
return ""
def _first_chat_model(models) -> str:
"""First model that isn't an embedding/tts/etc. — falls back to models[0]."""
for m in (models or []):
if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
return m
return (models[0] if models else "")
def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
"""Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
owner = owner or getattr(sess, "owner", None) or None
url, model, headers = resolve_endpoint(
"research",
fallback_url=sess.endpoint_url,
fallback_model=sess.model,
fallback_headers=sess.headers,
owner=owner,
)
return url, model, headers
def _owned_enabled_endpoint(db, owner, endpoint_id=None):
"""An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
None if nothing visible matches.
Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
owner = private, "the model picker only shows the endpoint to that user") and
holds a decrypted `api_key`. /api/research/start feeds the resolved row's
api_key + base_url into research_handler.start_research(llm_endpoint=,
llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
via the bare first-enabled fallback — would let a research-privileged user
spend ANOTHER user's API key/quota and reach whatever internal base_url they
configured. Mirrors webhook_routes._first_enabled_endpoint and
session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
legacy mode).
"""
from src.database import ModelEndpoint
from src.auth_helpers import owner_filter
q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
if endpoint_id:
q = q.filter(ModelEndpoint.id == endpoint_id)
return owner_filter(q, ModelEndpoint, owner).first()
def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
"""Resolve a ModelEndpoint row into (chat_url, model, headers).
Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
panel-selected research endpoints. ChatGPT Subscription endpoints keep
OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
"""
from src.endpoint_resolver import (
build_chat_url,
build_headers,
resolve_endpoint_runtime as resolve_model_endpoint_runtime,
)
try:
base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
except Exception as e:
logger.warning("Could not resolve endpoint credentials for research: %s", e)
return None
ep_model = (model or "").strip()
if not ep_model:
try:
models = json.loads(ep.cached_models) if ep.cached_models else []
if models:
ep_model = _first_chat_model(models)
except Exception:
pass
if not ep_model:
return None
return build_chat_url(base), ep_model, build_headers(api_key, base)
def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
router = APIRouter(tags=["research"])
def _require_user(request: Request) -> str:
"""All research endpoints require an authenticated user. Research
data isn't owner-scoped in the on-disk JSON yet, so we at least
block anonymous access. Multi-tenant deploys should additionally
verify the session belongs to this user."""
user = get_current_user(request)
if not user:
if _auth_disabled():
return ""
raise HTTPException(401, "Not authenticated")
return user
def _owns_in_memory(session_id: str, user: str) -> bool:
"""Ownership check for an in-flight (in-memory) research task.
Falls back to the on-disk JSON if the task has already finished."""
entry = research_handler._active_tasks.get(session_id)
if entry is not None:
return entry.get("owner", "") == user
# Task no longer in memory — check the persisted JSON.
try:
return _find_owned_research_path(session_id, user) is not None
except HTTPException:
return False
def _require_owned_or_active_research_path(session_id: str, user: str) -> Path | None:
"""Validate ownership once and return the completed on-disk path.
Active running research has no completed disk path yet. Completed
tasks can remain in _active_tasks after persistence, so prefer their
owned disk path when available. Completed disk lookups still reuse the
path after the ownership gate.
"""
entry = research_handler._active_tasks.get(session_id)
if entry is not None:
if entry.get("owner", "") != user:
raise HTTPException(404, "No research found for this session")
if entry.get("status") != "running":
path = _find_owned_research_path(session_id, user)
if path is not None:
return path
return None
path = _find_owned_research_path(session_id, user)
if path is None:
raise HTTPException(404, "No research found for this session")
return path
@router.get("/api/research/active")
async def research_active(request: Request):
"""List all currently active (running) research tasks."""
user = _require_user(request)
active = []
for sid, entry in research_handler._active_tasks.items():
# SECURITY: only show this user's running tasks.
if entry.get("owner", "") != user:
continue
if entry.get("status") == "running":
active.append({
"session_id": sid,
"query": entry.get("query", ""),
"status": "running",
"progress": entry.get("progress", {}),
"started_at": entry.get("started_at", 0),
})
return {"active": active}
@router.get("/api/research/status/{session_id}")
async def research_status(session_id: str, request: Request):
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
status = research_handler.get_status(session_id)
if status is None:
raise HTTPException(404, "No research found for this session")
return status
@router.post("/api/research/cancel/{session_id}")
async def research_cancel(session_id: str, request: Request):
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
cancelled = research_handler.cancel_research(session_id)
return {"cancelled": cancelled}
@router.post("/api/research/result/{session_id}")
async def research_result(session_id: str, request: Request):
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research result available")
result = research_handler.get_result(session_id)
if result is None:
raise HTTPException(404, "No research result available")
sources = research_handler.get_sources(session_id) or []
raw_findings = research_handler.get_raw_findings(session_id) or []
research_handler.clear_result(session_id)
return {"result": result, "sources": sources, "raw_findings": raw_findings}
def _assert_owns_research(session_id: str, user: str) -> None:
"""404-not-403 ownership gate for a research session's on-disk JSON.
Use BEFORE returning any data or mutating the file."""
path = _require_research_path(session_id)
try:
owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
except Exception:
raise HTTPException(404, "Research not found")
if owner != user:
raise HTTPException(404, "Research not found")
@router.get("/api/research/report/{session_id}")
async def research_report(session_id: str, request: Request):
"""Serve the visual HTML report for a completed research session."""
user = _require_user(request)
_validate_session_id(session_id)
_assert_owns_research(session_id, user)
logger.info(f"Visual report requested for session {session_id}")
try:
html_content = research_handler.get_report_html(session_id)
except Exception as e:
logger.error(f"Visual report generation error: {e}", exc_info=True)
raise HTTPException(500, f"Report generation failed: {e}")
if html_content is None:
logger.warning(f"No report data found for session {session_id}")
raise HTTPException(404, "No visual report available for this session")
return HTMLResponse(content=html_content)
class HideImageRequest(BaseModel):
url: str
@router.post("/api/research/{session_id}/hide-image")
async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
"""Mark an image URL as hidden for this research's visual report.
Persisted to the research JSON so subsequent /report renders skip it."""
user = _require_user(request)
_validate_session_id(session_id)
_assert_owns_research(session_id, user)
ok = research_handler.hide_image(session_id, body.url)
if not ok:
raise HTTPException(404, "Research not found")
return {"ok": True}
@router.post("/api/research/{session_id}/unhide-images")
async def research_unhide_images(session_id: str, request: Request):
"""Clear the hidden-images list for a research session."""
user = _require_user(request)
_validate_session_id(session_id)
_assert_owns_research(session_id, user)
ok = research_handler.unhide_all_images(session_id)
if not ok:
raise HTTPException(404, "Research not found")
return {"ok": True}
@router.get("/api/research/library")
async def research_library(
request: Request,
search: Optional[str] = Query(None),
sort: str = Query("recent"),
limit: int = Query(50),
archived: bool = Query(False),
):
user = _require_user(request)
"""List all completed research for the Library panel."""
data_dir = Path(DEEP_RESEARCH_DIR)
items = []
for p in data_dir.glob("*.json"):
try:
d = json.loads(p.read_text(encoding="utf-8"))
# SECURITY: only show research belonging to this user. Legacy
# JSONs without an `owner` field are hidden — auth was the only
# gate before, so every user saw every other user's reports.
if d.get("owner") != user:
continue
# Archived view shows ONLY archived reports; default hides them.
if bool(d.get("archived")) != archived:
continue
query = d.get("query", "")
if search and search.lower() not in query.lower():
continue
sources = d.get("sources", [])
items.append({
"id": p.stem,
"query": query,
"category": d.get("category") or "",
"source_count": len(sources),
"status": d.get("status", "done"),
"duration": d.get("stats", {}).get("Duration", ""),
"rounds": d.get("stats", {}).get("Rounds", ""),
"started_at": d.get("started_at", 0),
"completed_at": d.get("completed_at", 0),
"archived": bool(d.get("archived")),
"thumbnail": _research_thumbnail(d),
})
except Exception:
continue
# Sort
if sort == "recent":
items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
elif sort == "oldest":
items.sort(key=lambda x: x["completed_at"] or 0)
elif sort == "most-messages":
items.sort(key=lambda x: x["source_count"], reverse=True)
elif sort == "alpha":
items.sort(key=lambda x: x["query"].lower())
return {"research": items[:limit], "total": len(items)}
@router.get("/api/research/detail/{session_id}")
async def research_detail(session_id: str, request: Request):
"""Return the full JSON for a single research result — sources,
summary, stats — used by the Library preview panel."""
user = _require_user(request)
_validate_session_id(session_id)
path = _require_research_path(session_id)
try:
data = json.loads(path.read_text(encoding="utf-8"))
except Exception as e:
raise HTTPException(500, f"Failed to read research: {e}")
# SECURITY: 404 (not 403) so we don't leak that the report exists.
if data.get("owner") != user:
raise HTTPException(404, "Research not found")
return data
@router.post("/api/research/{session_id}/archive")
async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
"""Soft-archive / restore a research report (sets `archived` in its JSON)."""
user = _require_user(request)
_validate_session_id(session_id)
path = _require_research_path(session_id)
try:
data = json.loads(path.read_text(encoding="utf-8"))
if data.get("owner") != user:
raise HTTPException(404, "Research not found")
data["archived"] = bool(archived)
path.write_text(json.dumps(data), encoding="utf-8")
except HTTPException:
raise
except Exception as e:
raise HTTPException(500, f"Failed to update research: {e}")
return {"ok": True, "id": session_id, "archived": bool(archived)}
@router.delete("/api/research/{session_id}")
async def research_delete(session_id: str, request: Request):
"""Delete a research result from disk."""
user = _require_user(request)
_validate_session_id(session_id)
json_path = _find_research_path(session_id)
deleted = False
if json_path is not None:
# SECURITY: verify ownership before letting the caller delete it.
try:
data = json.loads(json_path.read_text(encoding="utf-8"))
if data.get("owner") != user:
raise HTTPException(404, "Research not found")
except HTTPException:
raise
except Exception:
raise HTTPException(404, "Research not found")
json_path.unlink()
deleted = True
return {"deleted": deleted}
# ------------------------------------------------------------------
# Panel endpoints — launch research without a chat session
# ------------------------------------------------------------------
class ResearchStartRequest(BaseModel):
query: str
# max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
max_rounds: int = Field(default=0, ge=0, le=20)
search_provider: Optional[str] = None
endpoint_id: Optional[str] = None
model: Optional[str] = None
max_time: int = Field(default=300, ge=60, le=1800)
extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
category: Optional[str] = None
@router.post("/api/research/start")
async def research_start(body: ResearchStartRequest, request: Request):
"""Launch a research job from the dedicated panel."""
from src.auth_helpers import require_privilege
user = require_privilege(request, "can_use_research")
if user == INTERNAL_TOOL_USER:
tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
if tool_owner and tool_owner not in RESERVED_USERNAMES:
auth_mgr = getattr(request.app.state, "auth_manager", None)
if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
try:
privs = auth_mgr.get_privileges(tool_owner) or {}
if not privs.get("can_use_research", True):
raise HTTPException(403, f"Your account is not allowed to can use research.")
except HTTPException:
raise
except Exception:
pass
user = tool_owner
session_id = f"rp-{uuid.uuid4().hex[:12]}"
if body.endpoint_id:
from src.database import SessionLocal
db = SessionLocal()
try:
# Owner-scoped: never resolve another user's private endpoint
# (and its decrypted api_key / internal base_url). A scoped miss
# reads as 404 so the endpoint's existence isn't revealed.
ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
if not ep:
raise HTTPException(404, "Endpoint not found or disabled")
resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
if not resolved:
raise HTTPException(400, "Endpoint is not configured with a usable model.")
ep_url, ep_model, ep_headers = resolved
finally:
db.close()
else:
ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
if not ep_url:
ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
# When neither research nor utility is configured, use the user's
# configured DEFAULT model (default_endpoint_id/default_model) rather
# than arbitrarily grabbing the first enabled endpoint's first model
# (which surfaced gpt-3.5). "Default" should mean the default model.
if not ep_url:
ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
if not ep_url:
ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
if not ep_url:
from src.database import SessionLocal
db = SessionLocal()
try:
# Owner-scoped first-enabled fallback: the caller's own rows
# + legacy null-owner shared rows only — never borrow another
# user's private endpoint/api_key. Same fix as the
# /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
ep = _owned_enabled_endpoint(db, user)
if ep:
resolved = _resolve_endpoint_runtime(ep, owner=user)
if resolved:
ep_url, ep_model, ep_headers = resolved
finally:
db.close()
if not ep_url:
raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
if body.model:
ep_model = body.model
# max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
research_handler.start_research(
session_id=session_id,
query=body.query,
llm_endpoint=ep_url,
llm_model=ep_model,
max_time=body.max_time,
llm_headers=ep_headers,
max_rounds=effective_max_rounds,
search_provider=body.search_provider or None,
category=body.category or None,
extraction_timeout=body.extraction_timeout,
extraction_concurrency=body.extraction_concurrency,
owner=user,
)
return {"session_id": session_id, "status": "running", "query": body.query}
@router.get("/api/research/stream/{session_id}")
async def research_stream(session_id: str, request: Request):
"""SSE stream of research progress events."""
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
async def _generate():
last_progress = None
while True:
status = research_handler.get_status(session_id)
if status is None:
yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
return
st = status.get("status", "")
progress = status.get("progress", {})
if progress != last_progress:
last_progress = progress
yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
if st != "running":
final = {'status': st, 'final': True}
task = research_handler._active_tasks.get(session_id, {})
if st == "error" and task.get("result"):
final['error'] = str(task["result"])[:500]
yield f"data: {json.dumps(final)}\n\n"
return
await asyncio.sleep(1.5)
return StreamingResponse(
_generate(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.post("/api/research/result-peek/{session_id}")
async def research_result_peek(session_id: str, request: Request):
"""Get research result without clearing it (for panel use)."""
user = _require_user(request)
_validate_session_id(session_id)
owned_disk_path = _require_owned_or_active_research_path(session_id, user)
result = research_handler.get_result(session_id)
if result is None:
p = owned_disk_path
if p is not None:
d = json.loads(p.read_text(encoding="utf-8"))
return {
"result": d.get("result", ""),
"sources": d.get("sources", []),
"raw_findings": d.get("raw_findings", []),
"category": d.get("category") or "",
}
raise HTTPException(404, "No research result available")
sources = research_handler.get_sources(session_id) or []
raw_findings = research_handler.get_raw_findings(session_id) or []
return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
@router.post("/api/research/spinoff/{session_id}")
async def research_spinoff(session_id: str, request: Request):
"""Create a new chat session pre-seeded with this research as context.
Reads the persisted research result + sources for `session_id`, creates
a fresh session (inheriting endpoint/model/headers from the source
session if available, otherwise from the resolved chat endpoint), and
injects a single system message containing the report and sources so
the user can ask follow-up questions in a clean conversation.
"""
user = _require_user(request)
_validate_session_id(session_id)
# SECURITY: gate on ownership before reading the persisted research —
# otherwise any authenticated user could spin off (and thereby read)
# another user's report by guessing its session ID. Mirrors every other
# endpoint in this file (see result_peek above).
owned_disk_path = _require_owned_or_active_research_path(session_id, user)
if session_manager is None:
raise HTTPException(500, "session_manager not configured")
# Load research data — prefer in-memory result, fall back to disk
result = research_handler.get_result(session_id)
sources = research_handler.get_sources(session_id) or []
query = ""
path = owned_disk_path
if path is not None:
try:
disk = json.loads(path.read_text(encoding="utf-8"))
if not result:
result = disk.get("result")
if not sources:
sources = disk.get("sources", []) or []
query = disk.get("query", "") or ""
except Exception as e:
logger.warning(f"Could not read research JSON for spinoff: {e}")
if not result:
raise HTTPException(404, "No research result available for this session")
# Inherit endpoint/model/headers from the source session when possible.
# For panel-launched research (rp-* IDs), there is no chat session, so
# fall back through the same chain as /api/research/start: research →
# utility → first enabled endpoint in the DB.
ep_url, ep_model, ep_headers = "", "", {}
try:
src_sess = session_manager.get_session(session_id)
ep_url = src_sess.endpoint_url or ""
ep_model = src_sess.model or ""
ep_headers = dict(src_sess.headers or {})
except KeyError:
pass
def _merge(r_url, r_model, r_headers):
nonlocal ep_url, ep_model, ep_headers
if not ep_url and r_url:
ep_url = r_url
if not ep_model and r_model:
ep_model = r_model
if not ep_headers and r_headers:
ep_headers = dict(r_headers)
if not ep_url or not ep_model:
_merge(*resolve_endpoint("chat", owner=user))
if not ep_url or not ep_model:
_merge(*resolve_endpoint("research", owner=user))
if not ep_url or not ep_model:
_merge(*resolve_endpoint("utility", owner=user))
if not ep_url or not ep_model:
# Last resort: this user's enabled endpoint, plus legacy shared rows.
from src.database import SessionLocal
from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
db = SessionLocal()
try:
ep = _owned_enabled_endpoint(db, user)
if ep:
base = normalize_base(ep.base_url)
fallback_url = build_chat_url(base)
fallback_headers = build_headers(ep.api_key, base)
fallback_model = ""
if ep.cached_models:
try:
models = json.loads(ep.cached_models)
if models:
fallback_model = _first_chat_model(models)
except Exception:
pass
_merge(fallback_url, fallback_model, fallback_headers)
finally:
db.close()
if not ep_url or not ep_model:
raise HTTPException(400, "No endpoint configured — add one in Settings first")
# Create new session
new_sid = str(uuid.uuid4())
title_query = (query or "research").strip()
if len(title_query) > 60:
title_query = title_query[:57] + "…"
new_name = f"Follow-up: {title_query}"
new_sess = session_manager.create_session(
session_id=new_sid,
name=new_name,
endpoint_url=ep_url,
model=ep_model,
rag=False,
owner=user,
)
if ep_headers:
new_sess.headers = ep_headers
session_manager.save_sessions()
try:
from src.event_bus import fire_event
fire_event("session_created", user)
except Exception:
logger.debug("session_created event dispatch failed", exc_info=True)
# Build the priming system message — report only, no sources injected.
# The user can open the visual report for source details; keeping sources
# out of the chat context saves tokens and avoids the AI fabricating
# citations.
date_str = datetime.utcnow().strftime("%Y-%m-%d")
primer = (
f"[Research context — {date_str}]\n\n"
f"The user previously ran a deep research investigation. Use the "
f"report below as your primary knowledge base when answering "
f"follow-up questions. If the user asks something not covered, "
f"say so plainly rather than guessing.\n\n"
f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
f"=== REPORT ===\n{result}"
)
from core.models import ChatMessage
new_sess.add_message(ChatMessage(
role="system",
content=primer,
metadata={"research_spinoff_from": session_id},
))
session_manager.save_sessions()
return {
"session_id": new_sid,
"name": new_name,
"source_count": len(sources),
}
return router
routes/research_routes.py
+672 -13
View File
@@ -1,17 +1,676 @@
"""Backward-compat shim — canonical location is routes/research/research_routes.py.
"""Research background task routes — /api/research/*."""
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.research_routes``, ``from routes.research_routes import X``,
``importlib.import_module("routes.research_routes")``, and
``monkeypatch.setattr("routes.research_routes.ATTR", ...)`` (string-targeted
patch used by ``test_research_owner_scope_routes.py``) all operate on the
*same* object the application actually uses. Keeps existing import paths
working after slice 2b (#4082/#4071). Source-introspection tests read the
canonical file by path.
"""
import asyncio
import json
import logging
import re
import uuid
from datetime import datetime
from pathlib import Path
from typing import Optional
import sys as _sys
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import HTMLResponse, StreamingResponse
from pydantic import BaseModel, Field
from src.endpoint_resolver import resolve_endpoint
from src.auth_helpers import _auth_disabled, get_current_user
from src.constants import DEEP_RESEARCH_DIR
from routes.research import research_routes as _canonical # noqa: F401
_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
_sys.modules[__name__] = _canonical
logger = logging.getLogger(__name__)
# Model-name substrings that are NOT chat/generation models — research must
# never pick these as its model. An OpenAI-style endpoint often lists
# `text-embedding-ada-002` etc. first in its model list, which is why research
# was failing with "Cannot reach model 'text-embedding-ada-002'".
_NON_CHAT_MODEL = (
"text-embedding", "embedding", "tts-", "whisper", "dall-e",
"moderation", "rerank", "reranker", "clip", "stable-diffusion",
)
def _first_chat_model(models) -> str:
"""First model that isn't an embedding/tts/etc. — falls back to models[0]."""
for m in (models or []):
if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
return m
return (models[0] if models else "")
def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
"""Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
owner = owner or getattr(sess, "owner", None) or None
url, model, headers = resolve_endpoint(
"research",
fallback_url=sess.endpoint_url,
fallback_model=sess.model,
fallback_headers=sess.headers,
owner=owner,
)
return url, model, headers
def _owned_enabled_endpoint(db, owner, endpoint_id=None):
"""An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
None if nothing visible matches.
Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
owner = private, "the model picker only shows the endpoint to that user") and
holds a decrypted `api_key`. /api/research/start feeds the resolved row's
api_key + base_url into research_handler.start_research(llm_endpoint=,
llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
via the bare first-enabled fallback — would let a research-privileged user
spend ANOTHER user's API key/quota and reach whatever internal base_url they
configured. Mirrors webhook_routes._first_enabled_endpoint and
session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
legacy mode).
"""
from src.database import ModelEndpoint
from src.auth_helpers import owner_filter
q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712
if endpoint_id:
q = q.filter(ModelEndpoint.id == endpoint_id)
return owner_filter(q, ModelEndpoint, owner).first()
def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
"""Resolve a ModelEndpoint row into (chat_url, model, headers).
Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
panel-selected research endpoints. ChatGPT Subscription endpoints keep
OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
"""
from src.endpoint_resolver import (
build_chat_url,
build_headers,
resolve_endpoint_runtime as resolve_model_endpoint_runtime,
)
try:
base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
except Exception as e:
logger.warning("Could not resolve endpoint credentials for research: %s", e)
return None
ep_model = (model or "").strip()
if not ep_model:
try:
models = json.loads(ep.cached_models) if ep.cached_models else []
if models:
ep_model = _first_chat_model(models)
except Exception:
pass
if not ep_model:
return None
return build_chat_url(base), ep_model, build_headers(api_key, base)
def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
router = APIRouter(tags=["research"])
def _require_user(request: Request) -> str:
"""All research endpoints require an authenticated user. Research
data isn't owner-scoped in the on-disk JSON yet, so we at least
block anonymous access. Multi-tenant deploys should additionally
verify the session belongs to this user."""
user = get_current_user(request)
if not user:
if _auth_disabled():
return ""
raise HTTPException(401, "Not authenticated")
return user
def _validate_session_id(session_id: str) -> None:
if not _SESSION_ID_RE.fullmatch(session_id):
raise HTTPException(400, "Invalid session ID format")
def _owns_in_memory(session_id: str, user: str) -> bool:
"""Ownership check for an in-flight (in-memory) research task.
Falls back to the on-disk JSON if the task has already finished."""
entry = research_handler._active_tasks.get(session_id)
if entry is not None:
return entry.get("owner", "") == user
# Task no longer in memory — check the persisted JSON.
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
return False
try:
return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
except Exception:
return False
@router.get("/api/research/active")
async def research_active(request: Request):
"""List all currently active (running) research tasks."""
user = _require_user(request)
active = []
for sid, entry in research_handler._active_tasks.items():
# SECURITY: only show this user's running tasks.
if entry.get("owner", "") != user:
continue
if entry.get("status") == "running":
active.append({
"session_id": sid,
"query": entry.get("query", ""),
"status": "running",
"progress": entry.get("progress", {}),
"started_at": entry.get("started_at", 0),
})
return {"active": active}
@router.get("/api/research/status/{session_id}")
async def research_status(session_id: str, request: Request):
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
status = research_handler.get_status(session_id)
if status is None:
raise HTTPException(404, "No research found for this session")
return status
@router.post("/api/research/cancel/{session_id}")
async def research_cancel(session_id: str, request: Request):
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
cancelled = research_handler.cancel_research(session_id)
return {"cancelled": cancelled}
@router.post("/api/research/result/{session_id}")
async def research_result(session_id: str, request: Request):
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research result available")
result = research_handler.get_result(session_id)
if result is None:
raise HTTPException(404, "No research result available")
sources = research_handler.get_sources(session_id) or []
raw_findings = research_handler.get_raw_findings(session_id) or []
research_handler.clear_result(session_id)
return {"result": result, "sources": sources, "raw_findings": raw_findings}
def _assert_owns_research(session_id: str, user: str) -> None:
"""404-not-403 ownership gate for a research session's on-disk JSON.
Use BEFORE returning any data or mutating the file."""
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
raise HTTPException(404, "Research not found")
try:
owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
except Exception:
raise HTTPException(404, "Research not found")
if owner != user:
raise HTTPException(404, "Research not found")
@router.get("/api/research/report/{session_id}")
async def research_report(session_id: str, request: Request):
"""Serve the visual HTML report for a completed research session."""
user = _require_user(request)
_validate_session_id(session_id)
_assert_owns_research(session_id, user)
logger.info(f"Visual report requested for session {session_id}")
try:
html_content = research_handler.get_report_html(session_id)
except Exception as e:
logger.error(f"Visual report generation error: {e}", exc_info=True)
raise HTTPException(500, f"Report generation failed: {e}")
if html_content is None:
logger.warning(f"No report data found for session {session_id}")
raise HTTPException(404, "No visual report available for this session")
return HTMLResponse(content=html_content)
class HideImageRequest(BaseModel):
url: str
@router.post("/api/research/{session_id}/hide-image")
async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
"""Mark an image URL as hidden for this research's visual report.
Persisted to the research JSON so subsequent /report renders skip it."""
user = _require_user(request)
_validate_session_id(session_id)
_assert_owns_research(session_id, user)
ok = research_handler.hide_image(session_id, body.url)
if not ok:
raise HTTPException(404, "Research not found")
return {"ok": True}
@router.post("/api/research/{session_id}/unhide-images")
async def research_unhide_images(session_id: str, request: Request):
"""Clear the hidden-images list for a research session."""
user = _require_user(request)
_validate_session_id(session_id)
_assert_owns_research(session_id, user)
ok = research_handler.unhide_all_images(session_id)
if not ok:
raise HTTPException(404, "Research not found")
return {"ok": True}
@router.get("/api/research/library")
async def research_library(
request: Request,
search: Optional[str] = Query(None),
sort: str = Query("recent"),
limit: int = Query(50),
archived: bool = Query(False),
):
user = _require_user(request)
"""List all completed research for the Library panel."""
data_dir = Path(DEEP_RESEARCH_DIR)
items = []
for p in data_dir.glob("*.json"):
try:
d = json.loads(p.read_text(encoding="utf-8"))
# SECURITY: only show research belonging to this user. Legacy
# JSONs without an `owner` field are hidden — auth was the only
# gate before, so every user saw every other user's reports.
if d.get("owner") != user:
continue
# Archived view shows ONLY archived reports; default hides them.
if bool(d.get("archived")) != archived:
continue
query = d.get("query", "")
if search and search.lower() not in query.lower():
continue
sources = d.get("sources", [])
items.append({
"id": p.stem,
"query": query,
"category": d.get("category") or "",
"source_count": len(sources),
"status": d.get("status", "done"),
"duration": d.get("stats", {}).get("Duration", ""),
"rounds": d.get("stats", {}).get("Rounds", ""),
"started_at": d.get("started_at", 0),
"completed_at": d.get("completed_at", 0),
"archived": bool(d.get("archived")),
})
except Exception:
continue
# Sort
if sort == "recent":
items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
elif sort == "oldest":
items.sort(key=lambda x: x["completed_at"] or 0)
elif sort == "most-messages":
items.sort(key=lambda x: x["source_count"], reverse=True)
elif sort == "alpha":
items.sort(key=lambda x: x["query"].lower())
return {"research": items[:limit], "total": len(items)}
@router.get("/api/research/detail/{session_id}")
async def research_detail(session_id: str, request: Request):
"""Return the full JSON for a single research result — sources,
summary, stats — used by the Library preview panel."""
user = _require_user(request)
_validate_session_id(session_id)
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
raise HTTPException(404, "Research not found")
try:
data = json.loads(path.read_text(encoding="utf-8"))
except Exception as e:
raise HTTPException(500, f"Failed to read research: {e}")
# SECURITY: 404 (not 403) so we don't leak that the report exists.
if data.get("owner") != user:
raise HTTPException(404, "Research not found")
return data
@router.post("/api/research/{session_id}/archive")
async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
"""Soft-archive / restore a research report (sets `archived` in its JSON)."""
user = _require_user(request)
_validate_session_id(session_id)
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
raise HTTPException(404, "Research not found")
try:
data = json.loads(path.read_text(encoding="utf-8"))
if data.get("owner") != user:
raise HTTPException(404, "Research not found")
data["archived"] = bool(archived)
path.write_text(json.dumps(data), encoding="utf-8")
except HTTPException:
raise
except Exception as e:
raise HTTPException(500, f"Failed to update research: {e}")
return {"ok": True, "id": session_id, "archived": bool(archived)}
@router.delete("/api/research/{session_id}")
async def research_delete(session_id: str, request: Request):
"""Delete a research result from disk."""
user = _require_user(request)
_validate_session_id(session_id)
data_dir = Path(DEEP_RESEARCH_DIR)
json_path = data_dir / f"{session_id}.json"
deleted = False
if json_path.exists():
# SECURITY: verify ownership before letting the caller delete it.
try:
data = json.loads(json_path.read_text(encoding="utf-8"))
if data.get("owner") != user:
raise HTTPException(404, "Research not found")
except HTTPException:
raise
except Exception:
raise HTTPException(404, "Research not found")
json_path.unlink()
deleted = True
return {"deleted": deleted}
# ------------------------------------------------------------------
# Panel endpoints — launch research without a chat session
# ------------------------------------------------------------------
class ResearchStartRequest(BaseModel):
query: str
# max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
max_rounds: int = Field(default=0, ge=0, le=20)
search_provider: Optional[str] = None
endpoint_id: Optional[str] = None
model: Optional[str] = None
max_time: int = Field(default=300, ge=60, le=1800)
extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
category: Optional[str] = None
@router.post("/api/research/start")
async def research_start(body: ResearchStartRequest, request: Request):
"""Launch a research job from the dedicated panel."""
from src.auth_helpers import require_privilege
user = require_privilege(request, "can_use_research")
if user == "internal-tool":
tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
if tool_owner and tool_owner not in {"internal-tool", "api", "demo", "system"}:
auth_mgr = getattr(request.app.state, "auth_manager", None)
if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
try:
privs = auth_mgr.get_privileges(tool_owner) or {}
if not privs.get("can_use_research", True):
raise HTTPException(403, f"Your account is not allowed to can use research.")
except HTTPException:
raise
except Exception:
pass
user = tool_owner
session_id = f"rp-{uuid.uuid4().hex[:12]}"
if body.endpoint_id:
from src.database import SessionLocal
db = SessionLocal()
try:
# Owner-scoped: never resolve another user's private endpoint
# (and its decrypted api_key / internal base_url). A scoped miss
# reads as 404 so the endpoint's existence isn't revealed.
ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
if not ep:
raise HTTPException(404, "Endpoint not found or disabled")
resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
if not resolved:
raise HTTPException(400, "Endpoint is not configured with a usable model.")
ep_url, ep_model, ep_headers = resolved
finally:
db.close()
else:
ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
if not ep_url:
ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
# When neither research nor utility is configured, use the user's
# configured DEFAULT model (default_endpoint_id/default_model) rather
# than arbitrarily grabbing the first enabled endpoint's first model
# (which surfaced gpt-3.5). "Default" should mean the default model.
if not ep_url:
ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
if not ep_url:
ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
if not ep_url:
from src.database import SessionLocal
db = SessionLocal()
try:
# Owner-scoped first-enabled fallback: the caller's own rows
# + legacy null-owner shared rows only — never borrow another
# user's private endpoint/api_key. Same fix as the
# /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
ep = _owned_enabled_endpoint(db, user)
if ep:
resolved = _resolve_endpoint_runtime(ep, owner=user)
if resolved:
ep_url, ep_model, ep_headers = resolved
finally:
db.close()
if not ep_url:
raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
if body.model:
ep_model = body.model
# max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
research_handler.start_research(
session_id=session_id,
query=body.query,
llm_endpoint=ep_url,
llm_model=ep_model,
max_time=body.max_time,
llm_headers=ep_headers,
max_rounds=effective_max_rounds,
search_provider=body.search_provider or None,
category=body.category or None,
extraction_timeout=body.extraction_timeout,
extraction_concurrency=body.extraction_concurrency,
owner=user,
)
return {"session_id": session_id, "status": "running", "query": body.query}
@router.get("/api/research/stream/{session_id}")
async def research_stream(session_id: str, request: Request):
"""SSE stream of research progress events."""
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
async def _generate():
last_progress = None
while True:
status = research_handler.get_status(session_id)
if status is None:
yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
return
st = status.get("status", "")
progress = status.get("progress", {})
if progress != last_progress:
last_progress = progress
yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
if st != "running":
final = {'status': st, 'final': True}
task = research_handler._active_tasks.get(session_id, {})
if st == "error" and task.get("result"):
final['error'] = str(task["result"])[:500]
yield f"data: {json.dumps(final)}\n\n"
return
await asyncio.sleep(1.5)
return StreamingResponse(
_generate(),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
@router.post("/api/research/result-peek/{session_id}")
async def research_result_peek(session_id: str, request: Request):
"""Get research result without clearing it (for panel use)."""
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
result = research_handler.get_result(session_id)
if result is None:
p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if p.exists():
d = json.loads(p.read_text(encoding="utf-8"))
return {
"result": d.get("result", ""),
"sources": d.get("sources", []),
"raw_findings": d.get("raw_findings", []),
"category": d.get("category") or "",
}
raise HTTPException(404, "No research result available")
sources = research_handler.get_sources(session_id) or []
raw_findings = research_handler.get_raw_findings(session_id) or []
return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
@router.post("/api/research/spinoff/{session_id}")
async def research_spinoff(session_id: str, request: Request):
"""Create a new chat session pre-seeded with this research as context.
Reads the persisted research result + sources for `session_id`, creates
a fresh session (inheriting endpoint/model/headers from the source
session if available, otherwise from the resolved chat endpoint), and
injects a single system message containing the report and sources so
the user can ask follow-up questions in a clean conversation.
"""
user = _require_user(request)
_validate_session_id(session_id)
# SECURITY: gate on ownership before reading the persisted research —
# otherwise any authenticated user could spin off (and thereby read)
# another user's report by guessing its session ID. Mirrors every other
# endpoint in this file (see result_peek above).
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
if session_manager is None:
raise HTTPException(500, "session_manager not configured")
# Load research data — prefer in-memory result, fall back to disk
result = research_handler.get_result(session_id)
sources = research_handler.get_sources(session_id) or []
query = ""
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if path.exists():
try:
disk = json.loads(path.read_text(encoding="utf-8"))
if not result:
result = disk.get("result")
if not sources:
sources = disk.get("sources", []) or []
query = disk.get("query", "") or ""
except Exception as e:
logger.warning(f"Could not read research JSON for spinoff: {e}")
if not result:
raise HTTPException(404, "No research result available for this session")
# Inherit endpoint/model/headers from the source session when possible.
# For panel-launched research (rp-* IDs), there is no chat session, so
# fall back through the same chain as /api/research/start: research →
# utility → first enabled endpoint in the DB.
ep_url, ep_model, ep_headers = "", "", {}
try:
src_sess = session_manager.get_session(session_id)
ep_url = src_sess.endpoint_url or ""
ep_model = src_sess.model or ""
ep_headers = dict(src_sess.headers or {})
except KeyError:
pass
def _merge(r_url, r_model, r_headers):
nonlocal ep_url, ep_model, ep_headers
if not ep_url and r_url:
ep_url = r_url
if not ep_model and r_model:
ep_model = r_model
if not ep_headers and r_headers:
ep_headers = dict(r_headers)
if not ep_url or not ep_model:
_merge(*resolve_endpoint("chat", owner=user))
if not ep_url or not ep_model:
_merge(*resolve_endpoint("research", owner=user))
if not ep_url or not ep_model:
_merge(*resolve_endpoint("utility", owner=user))
if not ep_url or not ep_model:
# Last resort: this user's enabled endpoint, plus legacy shared rows.
from src.database import SessionLocal
from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
db = SessionLocal()
try:
ep = _owned_enabled_endpoint(db, user)
if ep:
base = normalize_base(ep.base_url)
fallback_url = build_chat_url(base)
fallback_headers = build_headers(ep.api_key, base)
fallback_model = ""
if ep.cached_models:
try:
models = json.loads(ep.cached_models)
if models:
fallback_model = _first_chat_model(models)
except Exception:
pass
_merge(fallback_url, fallback_model, fallback_headers)
finally:
db.close()
if not ep_url or not ep_model:
raise HTTPException(400, "No endpoint configured — add one in Settings first")
# Create new session
new_sid = str(uuid.uuid4())
title_query = (query or "research").strip()
if len(title_query) > 60:
title_query = title_query[:57] + "…"
new_name = f"Follow-up: {title_query}"
new_sess = session_manager.create_session(
session_id=new_sid,
name=new_name,
endpoint_url=ep_url,
model=ep_model,
rag=False,
owner=user,
)
if ep_headers:
new_sess.headers = ep_headers
session_manager.save_sessions()
try:
from src.event_bus import fire_event
fire_event("session_created", user)
except Exception:
logger.debug("session_created event dispatch failed", exc_info=True)
# Build the priming system message — report only, no sources injected.
# The user can open the visual report for source details; keeping sources
# out of the chat context saves tokens and avoids the AI fabricating
# citations.
date_str = datetime.utcnow().strftime("%Y-%m-%d")
primer = (
f"[Research context — {date_str}]\n\n"
f"The user previously ran a deep research investigation. Use the "
f"report below as your primary knowledge base when answering "
f"follow-up questions. If the user asks something not covered, "
f"say so plainly rather than guessing.\n\n"
f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
f"=== REPORT ===\n{result}"
)
from core.models import ChatMessage
new_sess.add_message(ChatMessage(
role="system",
content=primer,
metadata={"research_spinoff_from": session_id},
))
session_manager.save_sessions()
return {
"session_id": new_sid,
"name": new_name,
"source_count": len(sources),
}
return router
routes/session_routes.py
+18 -52
View File
@@ -11,9 +11,8 @@ from core.session_manager import SessionManager
from core.models import ChatMessage
from src.request_models import SessionResponse
from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive
from src.auth_helpers import effective_user, _auth_disabled, owner_filter
from src.auth_helpers import get_current_user, effective_user, _auth_disabled, owner_filter
from src.session_actions import is_session_recently_active
from src.upload_handler import reserve_message_upload_references
def _sanitize_export_filename(name: str) -> str:
@@ -163,7 +162,7 @@ def _persist_session_headers(session_id: str, headers: dict | None) -> None:
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.headers = headers or {}
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
db.commit()
except Exception:
db.rollback()
@@ -204,16 +203,10 @@ def _pick_endpoint_for_sort(owner=None):
return url, model, headers
return None, None, None
def setup_session_routes(
session_manager: SessionManager,
config: dict,
webhook_manager=None,
upload_handler=None,
):
def setup_session_routes(session_manager: SessionManager, config: dict, webhook_manager=None):
"""Setup session routes with the provided manager and config"""
REQUEST_TIMEOUT = config.get("REQUEST_TIMEOUT", 20)
SESSION_MODEL_VALIDATION_TIMEOUT = min(float(REQUEST_TIMEOUT or 20), 3.0)
OPENAI_API_KEY = config.get("OPENAI_API_KEY")
SESSIONS_FILE = config.get("SESSIONS_FILE")
@@ -230,8 +223,8 @@ def setup_session_routes(
# purge exists only to catch ghosts the frontend missed (tab close,
# crash). Only clean up rows old enough to be definitely orphaned.
try:
from datetime import timedelta as _td
_cutoff = utcnow_naive() - _td(minutes=10)
from datetime import datetime as _dt, timedelta as _td
_cutoff = _dt.utcnow() - _td(minutes=10)
_purge_db = SessionLocal()
try:
from core.database import ChatMessage as _DbMsg
@@ -335,7 +328,7 @@ def setup_session_routes(
endpoint_id: str = Form(""),
):
skip_val = str(skip_validation).lower() == "true"
user = effective_user(request)
user = get_current_user(request)
endpoint_api_key = ""
endpoint_base_url = ""
_reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url)
@@ -381,7 +374,7 @@ def setup_session_routes(
from src.llm_core import list_model_ids
ids = list_model_ids(
endpoint_url,
timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
timeout=REQUEST_TIMEOUT,
headers=validation_headers,
owner=user,
endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -401,7 +394,7 @@ def setup_session_routes(
req_base = _os.path.basename(model_to_use.rstrip("/"))
avail = list_model_ids(
endpoint_url,
timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
timeout=REQUEST_TIMEOUT,
headers=validation_headers,
owner=user,
endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -477,14 +470,14 @@ def setup_session_routes(
db_session = db.query(DbSession).filter(DbSession.id == sid).first()
if db_session:
db_session.folder = folder if folder else None
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
db.commit()
result["folder"] = folder if folder else None
finally:
db.close()
# Switch model/endpoint mid-session
if model is not None and endpoint_url is not None:
user = effective_user(request)
user = get_current_user(request)
_reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url)
endpoint_api_key = ""
endpoint_base_url = ""
@@ -524,7 +517,7 @@ def setup_session_routes(
db_session.model = model
db_session.endpoint_url = endpoint_url
db_session.headers = session.headers or {}
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
db.commit()
finally:
db.close()
@@ -543,22 +536,6 @@ def setup_session_routes(
body = await request.json()
messages = body.get("messages", [])
from core.models import ChatMessage
owner = effective_user(request)
try:
for message in messages:
missing_id = reserve_message_upload_references(
upload_handler,
owner,
message.get("content"),
message.get("metadata"),
)
if missing_id:
raise HTTPException(
409,
f"Referenced upload is no longer available: {missing_id}",
)
except (AttributeError, TypeError, ValueError) as exc:
raise HTTPException(400, "Invalid message attachment metadata") from exc
for m in messages:
sess.add_message(ChatMessage(m["role"], m["content"], metadata=m.get("metadata")))
session_manager.save_sessions()
@@ -669,7 +646,7 @@ def setup_session_routes(
db_session = db.query(DbSession).filter(DbSession.id == sid).first()
if db_session:
db_session.archived = True
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
db.commit()
# Update in memory if it exists
@@ -703,7 +680,7 @@ def setup_session_routes(
if not db_session:
raise HTTPException(404, f"Session {sid} not found")
db_session.archived = False
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
db.commit()
# Reload into session manager so it appears in the active list
try:
@@ -913,7 +890,7 @@ def setup_session_routes(
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.is_important = important
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
db.commit()
# Update in memory if it exists
@@ -1002,7 +979,7 @@ def setup_session_routes(
metadata={
"compacted": True,
"summarized_count": len(older),
"timestamp": utcnow_naive().isoformat(),
"timestamp": datetime.utcnow().isoformat(),
},
)
new_history = [summary_msg] + recent
@@ -1027,7 +1004,6 @@ def setup_session_routes(
"""
from src.llm_core import llm_call
user = effective_user(request)
single_user_mode = not user and _auth_disabled()
user_sessions = session_manager.get_sessions_for_user(user)
# Delete empty and throwaway sessions before sorting
@@ -1046,12 +1022,7 @@ def setup_session_routes(
}
_THROWAWAY_MAX_MESSAGES = 4 # only delete if <= this many messages
try:
rows_q = db.query(DbSession).filter(DbSession.archived == False)
if user:
rows_q = rows_q.filter(DbSession.owner == user)
elif not single_user_mode:
rows_q = rows_q.filter(DbSession.owner == user)
rows = rows_q.limit(2000).all()
rows = db.query(DbSession).filter(DbSession.archived == False, DbSession.owner == user).limit(2000).all()
folder_map = {r.id: r.folder for r in rows}
# Precompute per-session message counts in TWO aggregate queries
# instead of 1–3 queries PER session — with many chats the per-row
@@ -1271,15 +1242,10 @@ def setup_session_routes(
db = SessionLocal()
try:
for sid, folder_name in assignments.items():
db_session_q = db.query(DbSession).filter(DbSession.id == sid)
if user:
db_session_q = db_session_q.filter(DbSession.owner == user)
elif not single_user_mode:
db_session_q = db_session_q.filter(DbSession.owner == user)
db_session = db_session_q.first()
db_session = db.query(DbSession).filter(DbSession.id == sid, DbSession.owner == user).first()
if db_session:
db_session.folder = folder_name
db_session.updated_at = utcnow_naive()
db_session.updated_at = datetime.utcnow()
updated += 1
db.commit()
except Exception as e:
routes/shell_routes.py
+37 -484
View File
@@ -15,12 +15,6 @@ from collections import namedtuple
from pathlib import Path
from typing import Dict, Any
from core.platform_compat import IS_APPLE_SILICON, which_tool
from core.middleware import INTERNAL_TOOL_USER
from src.host_docker_access import (
HOST_DOCKER_ACCESS_HINT,
host_docker_access_enabled as _host_docker_access_enabled,
running_in_container as _running_in_container,
)
from src.optional_deps import prepare_optional_dependency_import
# POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
@@ -61,7 +55,7 @@ def _require_admin(request: Request):
# In-process tool loopback. The AuthMiddleware already validated the
# internal token + loopback client before setting this marker, so
# honour it here as admin-equivalent.
if user == INTERNAL_TOOL_USER:
if user == "internal-tool":
return
if not user or user == "api":
raise HTTPException(403, "Admin only")
@@ -108,17 +102,32 @@ logger = logging.getLogger(__name__)
PTY_SUPPORTED = pty is not None and fcntl is not None and hasattr(os, "setsid")
DOCKER_IN_CONTAINER_HINT = HOST_DOCKER_ACCESS_HINT
DOCKER_IN_CONTAINER_HINT = (
"Not available inside the Odysseus container by design. The image ships no "
"docker CLI and no host socket is mounted. Run Docker-backed launches on a "
"remote server, where docker is checked over SSH. Mounting /var/run/docker.sock "
"into the container would grant it host-root access, so only do that if you "
"accept that risk."
)
def _running_in_container(dockerenv_path="/.dockerenv", cgroup_path="/proc/1/cgroup"):
if os.path.exists(dockerenv_path):
return True
try:
with open(cgroup_path, "r", encoding="utf-8") as fh:
contents = fh.read()
except OSError:
return False
return any(token in contents for token in ("docker", "containerd", "kubepods"))
DockerRowStatus = namedtuple("DockerRowStatus", ["applicable", "install_hint"])
PackageUpdateStatus = namedtuple("PackageUpdateStatus", ["available", "note"])
def _docker_row_status(
*, on_remote, in_container, installed, default_hint, host_docker_access=False
):
local_docker_unavailable = not on_remote and in_container and not host_docker_access
def _docker_row_status(*, on_remote, in_container, installed, default_hint):
local_docker_unavailable = not on_remote and in_container and not installed
if local_docker_unavailable:
return DockerRowStatus(applicable=False, install_hint=DOCKER_IN_CONTAINER_HINT)
return DockerRowStatus(applicable=True, install_hint=default_hint)
@@ -164,8 +173,6 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
return bool(binaries.get("llama-server") or dists.get("llama-cpp-python"))
if name == "sglang":
return bool(dists.get("sglang") or modules.get("sglang", {}).get("real_module"))
if name == "mlx_lm":
return bool(dists.get("mlx-lm") or modules.get("mlx_lm", {}).get("real_module"))
if name == "diffusers":
return bool(
(dists.get("diffusers") or modules.get("diffusers", {}).get("real_module"))
@@ -212,10 +219,6 @@ def _package_status_note(name: str, probe: dict) -> str:
if _package_installed_from_probe(name, probe):
return f"diffusers {dists.get('diffusers', 'available')} with torch {dists.get('torch', 'available')}"
return "Diffusers serving needs both diffusers and torch."
if name == "mlx_lm":
if _package_installed_from_probe(name, probe):
return f"MLX LM {dists.get('mlx-lm', 'available')}"
return "MLX serving needs mlx-lm on an Apple Silicon Mac."
if name in dists:
return f"{name} {dists[name]}"
return ""
@@ -313,14 +316,12 @@ dist_names={{
'vllm':['vllm'],
'llama_cpp':['llama-cpp-python'],
'sglang':['sglang'],
'mlx_lm':['mlx-lm'],
'diffusers':['diffusers','torch'],
'hf_transfer':['hf-transfer','hf_transfer'],
}}
bin_names={{
'vllm':['vllm'],
'llama_cpp':['llama-server'],
'tmux':['tmux'],
}}
def add_user_install_bins_to_path():
@@ -329,12 +330,7 @@ def add_user_install_bins_to_path():
candidates.append(os.path.join(site.USER_BASE, 'bin'))
except Exception:
pass
candidates.append(os.path.expanduser('~/bin'))
candidates.append(os.path.expanduser('~/llama.cpp/build/bin'))
candidates.append(os.path.expanduser('~/llama.cpp/build-vulkan/bin'))
candidates.append(os.path.expanduser('~/.local/bin'))
candidates.append('/opt/homebrew/bin')
candidates.append('/usr/local/bin')
parts = os.environ.get('PATH', '').split(os.pathsep) if os.environ.get('PATH') else []
changed = False
for path in reversed([p for p in candidates if p]):
@@ -409,47 +405,6 @@ class ShellExecRequest(BaseModel):
use_tmux: bool = False # run in tmux session (survives browser disconnect)
_REMOTE_TMUX_PATH_PREFIX = 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; '
def _normalize_legacy_remote_tmux_exec(command: str) -> str:
"""Repair stale frontend Cookbook tmux SSH commands.
Older loaded JS sends `ssh host 'tmux capture-pane ...'`. On macOS/Homebrew
remotes, non-login SSH shells often lack /opt/homebrew/bin, so tmux is
installed but the capture/kill command returns nothing. Keep this narrowly
scoped to SSH commands whose remote shell starts with `tmux `.
"""
cmd = command or ""
if _REMOTE_TMUX_PATH_PREFIX in cmd or not cmd.lstrip().startswith("ssh "):
return cmd
try:
parts = shlex.split(cmd)
except Exception:
return cmd
if not parts or parts[0] != "ssh":
return cmd
remote_idx = -1
i = 1
while i < len(parts):
part = parts[i]
if part in {"-p", "-o", "-i", "-F", "-J", "-l", "-S", "-W", "-b", "-c", "-m"}:
i += 2
continue
if part.startswith("-"):
i += 1
continue
remote_idx = i
break
if remote_idx < 0 or remote_idx + 1 >= len(parts):
return cmd
remote_cmd = " ".join(parts[remote_idx + 1:]).strip()
if not remote_cmd.startswith("tmux "):
return cmd
repaired = parts[:remote_idx + 1] + [_REMOTE_TMUX_PATH_PREFIX + remote_cmd]
return shlex.join(repaired)
async def _create_shell(command: str, **kwargs):
"""Spawn a shell subprocess for `command`.
@@ -866,10 +821,6 @@ def setup_shell_routes() -> APIRouter:
if not cmd:
return {"stdout": "", "stderr": "No command provided", "exit_code": 1}
fixed_cmd = _normalize_legacy_remote_tmux_exec(cmd)
if fixed_cmd != cmd:
logger.info("Rewrote legacy remote tmux exec command with Homebrew PATH")
cmd = fixed_cmd
logger.info("User shell exec requested: length=%d", len(cmd))
result = await _exec_shell(
cmd, timeout=req.timeout if req.timeout is not None else EXEC_TIMEOUT
@@ -1010,84 +961,12 @@ def setup_shell_routes() -> APIRouter:
return StreamingResponse(generate(), media_type="text/event-stream")
def _os_id_from_release(text: str) -> str:
"""Map /etc/os-release contents to a canonical family for our matrix."""
if not text:
return ""
ids = []
for line in text.splitlines():
line = line.strip()
if line.startswith("ID=") or line.startswith("ID_LIKE="):
ids += line.split("=", 1)[1].strip().strip('"').split()
ids = [i.lower() for i in ids]
if any(x in ids for x in ("debian", "ubuntu", "linuxmint", "pop", "elementary")):
return "debian"
if any(x in ids for x in ("arch", "manjaro", "endeavouros", "cachyos", "garuda")):
return "arch"
if any(x in ids for x in ("fedora", "rhel", "centos", "rocky", "almalinux", "ol")):
return "fedora"
if "alpine" in ids:
return "alpine"
if any(x in ids for x in ("suse", "opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles")):
return "suse"
return ""
# Matrix lookup keyed on (os_family, backend) → (pkg_mgr_cmd_template, pkg_list_per_dep).
# Each `system_prereqs` name resolves to a list of OS-specific package
# names that get joined into the final `sudo apt install -y …` etc.
# command. Backend-specific extras (CUDA toolkit, ROCm, Vulkan headers)
# are added only when the detected backend needs them.
_PKG_NAMES = {
# canonical-name → {os_id: [actual_pkg_names_on_this_os]}
"cmake": {"debian": ["cmake"], "arch": ["cmake"], "fedora": ["cmake"], "alpine": ["cmake"], "suse": ["cmake"], "macos": ["cmake"]},
"build-essential": {"debian": ["build-essential"], "arch": ["base-devel"], "fedora": ["gcc", "gcc-c++", "make"], "alpine": ["build-base"], "suse": ["gcc-c++", "make"], "macos": []},
"g++": {"debian": ["g++"], "arch": ["gcc"], "fedora": ["gcc-c++"], "alpine": ["g++"], "suse": ["gcc-c++"], "macos": []},
"gcc": {"debian": ["gcc"], "arch": ["gcc"], "fedora": ["gcc"], "alpine": ["gcc"], "suse": ["gcc"], "macos": []},
"make": {"debian": ["make"], "arch": ["make"], "fedora": ["make"], "alpine": ["make"], "suse": ["make"], "macos": []},
"git": {"debian": ["git"], "arch": ["git"], "fedora": ["git"], "alpine": ["git"], "suse": ["git"], "macos": ["git"]},
"tmux": {"debian": ["tmux"], "arch": ["tmux"], "fedora": ["tmux"], "alpine": ["tmux"], "suse": ["tmux"], "macos": ["tmux"]},
}
_BACKEND_EXTRAS = {
"cuda": {"debian": ["nvidia-cuda-toolkit"], "arch": ["cuda"], "fedora": ["cuda-toolkit"], "alpine": [], "suse": ["cuda"], "macos": []},
"rocm": {"debian": ["rocm-dev"], "arch": ["rocm-hip-sdk"], "fedora": ["rocm-devel"], "alpine": [], "suse": ["rocm-dev"], "macos": []},
"vulkan": {"debian": ["libvulkan-dev", "vulkan-tools"], "arch": ["vulkan-headers", "vulkan-tools"], "fedora": ["vulkan-headers", "vulkan-tools"], "alpine": ["vulkan-loader-dev", "vulkan-tools"], "suse": ["vulkan-devel", "vulkan-tools"], "macos": []},
}
_PKG_MGR = {
"debian": "sudo apt install -y {pkgs}",
"arch": "sudo pacman -S --needed {pkgs}",
"fedora": "sudo dnf install -y {pkgs}",
"alpine": "sudo apk add {pkgs}",
"suse": "sudo zypper install -n {pkgs}",
"macos": "brew install {pkgs}",
}
def _install_cmd_for_target(os_id: str, backend: str, missing: list[str]) -> str:
"""Build a single OS+backend-aware install command for the missing prereqs."""
if not os_id or os_id not in _PKG_MGR:
return ""
pkgs: list[str] = []
seen: set[str] = set()
for m in missing:
for p in _PKG_NAMES.get(m, {}).get(os_id, []):
if p not in seen:
pkgs.append(p); seen.add(p)
# Add backend-specific extras only when the build would actually
# consume them (a CUDA toolkit isn't useful on a Vulkan box).
backend = (backend or "").lower()
for p in _BACKEND_EXTRAS.get(backend, {}).get(os_id, []):
if p not in seen:
pkgs.append(p); seen.add(p)
if not pkgs:
return ""
return _PKG_MGR[os_id].format(pkgs=" ".join(pkgs))
@router.get("/api/cookbook/packages")
async def list_packages(
request: Request,
host: str | None = None,
ssh_port: str | None = None,
venv: str | None = None,
backend: str | None = None,
):
"""Check which optional packages are installed.
@@ -1108,19 +987,8 @@ def setup_shell_routes() -> APIRouter:
importlib.invalidate_caches()
try:
user_site = site.getusersitepackages()
if user_site and os.path.isdir(user_site):
# Use addsitedir(), NOT a bare sys.path.append(). When a package
# is `pip install --user`'d at runtime (Cookbook → Install) the
# long-lived server process started before the user-site existed,
# so site never processed it — including its `.pth` hooks. On
# Python 3.12+ `distutils` is gone from stdlib and is only
# restored by setuptools' `distutils-precedence.pth`, which ships
# in user-site. basicsr (a realesrgan dep) does `import distutils`
# at import time, so a plain append left the package importable
# but `import distutils` failing → realesrgan probed as
# not-installed until a full process restart. addsitedir() replays
# the `.pth` files so the shim is active.
site.addsitedir(user_site)
if user_site and os.path.isdir(user_site) and user_site not in sys.path:
sys.path.append(user_site)
except Exception:
pass
if ssh_port and str(ssh_port).strip() not in ("", "22"):
@@ -1147,12 +1015,6 @@ def setup_shell_routes() -> APIRouter:
"kind": "system",
"install_hint": "Install Docker on the selected server and allow this user to run docker.",
},
# Note: cmake / gcc / git are not separate dependency rows —
# they're declared as `system_prereqs` on llama_cpp (and any
# other engine that compiles from source) so they appear as
# an inline status note on that engine's row instead of
# cluttering the panel with raw OS package names that aren't
# meaningful product-level dependencies on their own.
# ── LLM ── installs on GPU servers for model serving/downloading
{
"name": "hf_transfer",
@@ -1164,16 +1026,9 @@ def setup_shell_routes() -> APIRouter:
{
"name": "llama_cpp",
"pip": "llama-cpp-python[server]",
"desc": "Great for single-GPU or CPU inference with GGUF models",
"desc": "Serve GGUF models via llama.cpp",
"category": "LLM",
"target": "remote",
# Build-toolchain prereqs. Cookbook's launch bootstrap
# compiles llama-server from source when no prebuilt
# binary is present; without these the build aborts
# with `cmake: command not found`. Surfaced inline on
# this row so the user doesn't have to chase three
# separate OS-package rows.
"system_prereqs": ["cmake", "g++", "git"],
},
{
"name": "sglang",
@@ -1185,14 +1040,7 @@ def setup_shell_routes() -> APIRouter:
{
"name": "vllm",
"pip": "vllm",
"desc": "Great for high-throughput multi-GPU inference",
"category": "LLM",
"target": "remote",
},
{
"name": "mlx_lm",
"pip": "mlx-lm",
"desc": "Serve MLX-format models on Apple Silicon Macs",
"desc": "High-throughput LLM serving engine",
"category": "LLM",
"target": "remote",
},
@@ -1211,7 +1059,7 @@ def setup_shell_routes() -> APIRouter:
{
"name": "diffusers",
"pip": "diffusers[torch]",
"desc": "Image generation/editing pipelines (SD, Flux) with PyTorch",
"desc": "Image generation pipelines (SD, Flux) with PyTorch",
"category": "Image",
"target": "remote",
},
@@ -1255,7 +1103,6 @@ def setup_shell_routes() -> APIRouter:
# venv over SSH so a remote `pip install` actually reflects here.
remote_status: dict = {}
remote_details: dict = {}
remote_probe_error = ""
remote_names = [
p["name"]
for p in packages
@@ -1294,56 +1141,16 @@ def setup_shell_routes() -> APIRouter:
break
except ValueError as e:
raise HTTPException(400, str(e))
except Exception as e:
except Exception:
remote_status = {}
remote_probe_error = f"SSH package probe failed: {str(e)[:160]}"
if "llama_cpp" in remote_names:
try:
inner = (
'export PATH="$HOME/.local/bin:$HOME/bin:'
'$HOME/llama.cpp/build/bin:$HOME/llama.cpp/build-vulkan/bin:$PATH"; '
"command -v llama-server 2>/dev/null || true"
)
argv = _ssh_base_argv(host, ssh_port) + [inner]
proc = await asyncio.create_subprocess_exec(
*argv,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
out, _err = await asyncio.wait_for(proc.communicate(), timeout=8)
llama_server_path = out.decode("utf-8", errors="replace").strip().splitlines()
llama_server_path = llama_server_path[-1].strip() if llama_server_path else ""
if llama_server_path:
remote_status["llama_cpp"] = True
probe = remote_details.setdefault("llama_cpp", {})
if isinstance(probe, dict):
probe.setdefault("binaries", {})["llama-server"] = llama_server_path
except Exception as e:
if not remote_probe_error:
remote_probe_error = f"SSH llama-server probe failed: {str(e)[:160]}"
pass
# Union of system_names + every package's system_prereqs. Probing
# the prereqs alongside the main system deps in a single SSH call
# avoids a second round-trip per Cookbook → Dependencies refresh.
prereq_names: set[str] = set()
for p in packages:
for pr in p.get("system_prereqs") or []:
prereq_names.add(str(pr))
all_system_names = list(set(remote_system_names) | prereq_names)
# Detect the target's OS family + read /etc/os-release in the same
# SSH round-trip as the prereq probe — used downstream to render a
# single OS-specific install command per row instead of dumping
# every distro's syntax onto the user.
target_os_id: str = ""
if host and all_system_names:
if host and remote_system_names:
try:
checks = []
for name in all_system_names:
for name in remote_system_names:
qn = shlex.quote(name)
checks.append(
f"PATH=\"$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH\"; if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
f"if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
)
checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || { [ \"$(uname -s 2>/dev/null)\" = \"Darwin\" ] && echo ID=macos; } || true")
inner = " ; ".join(checks)
argv = _ssh_base_argv(host, ssh_port) + [inner]
proc = await asyncio.create_subprocess_exec(
@@ -1353,45 +1160,20 @@ def setup_shell_routes() -> APIRouter:
)
out, _err = await asyncio.wait_for(proc.communicate(), timeout=12)
txt = out.decode("utf-8", errors="replace").strip()
_section, _osrel_lines = "probe", []
for line in txt.splitlines():
if line.strip() == "---OSREL---":
_section = "osrel"; continue
if _section == "osrel":
_osrel_lines.append(line)
continue
name, sep, value = line.strip().partition("=")
if sep and name in all_system_names:
if sep and name in remote_system_names:
remote_status[name] = value == "1"
target_os_id = _os_id_from_release("\n".join(_osrel_lines))
except ValueError as e:
raise HTTPException(400, str(e))
except Exception as e:
if not remote_probe_error:
remote_probe_error = f"SSH system probe failed: {str(e)[:160]}"
pass
elif not host:
# Local target — probe in-process so the inline install command
# still appears in the dep panel when the cookbook container
# itself is the selected server.
try:
with open("/etc/os-release", encoding="utf-8") as f:
target_os_id = _os_id_from_release(f.read())
except Exception:
target_os_id = ""
if sys.platform == "darwin":
target_os_id = "macos"
pass
for pkg in packages:
on_remote = bool(host and pkg.get("target") == "remote")
probe = None
if on_remote:
if remote_probe_error and pkg["name"] not in remote_status:
pkg["installed"] = None
pkg["probe_error"] = remote_probe_error
pkg["status_note"] = remote_probe_error
else:
pkg["installed"] = bool(remote_status.get(pkg["name"], False))
pkg["installed"] = bool(remote_status.get(pkg["name"], False))
probe = remote_details.get(pkg["name"])
if isinstance(probe, dict):
pkg["details"] = probe
@@ -1440,116 +1222,13 @@ def setup_shell_routes() -> APIRouter:
pkg["installed"] = False
except importlib_metadata.PackageNotFoundError:
pkg["installed"] = False
except (Exception, SystemExit):
except Exception:
# Installed but crashes on import — e.g. a CUDA build of
# llama-cpp-python raising FileNotFoundError when the CUDA
# toolkit dir is absent, or rembg calling sys.exit(1) when no
# onnxruntime backend can be loaded. SystemExit is a
# BaseException, not Exception, so without catching it here a
# single sys.exit-on-import package escapes and takes down the
# whole packages panel / worker (the panel hangs forever). One
# broken optional package must not 500 — or hang — the entire
# panel; report it as not usable.
# toolkit dir is absent. One broken optional package must not
# 500 the entire packages panel; report it as not usable.
pkg["installed"] = False
# llama_cpp partial-state probe: when the package is installed
# but the wheel was built CPU-only AND the target has NVIDIA
# hardware, mark the row as partial (yellow/orange) with a
# one-click upgrade to the CUDA wheel. Without this the row
# reads "ready" green while inference runs at 3 tok/s on GPU
# silicon — actively misleading.
if pkg["name"] == "llama_cpp" and pkg.get("installed"):
_native_llama_server = bool(
isinstance(probe, dict)
and isinstance(probe.get("binaries"), dict)
and probe["binaries"].get("llama-server")
)
_gpu_capable = False
_has_nvidia_target = False
if _native_llama_server:
# Native llama-server is the launcher path Cookbook now
# prefers. Do not mark this as a CPU-only Python wheel just
# because llama-cpp-python is absent from the selected venv.
_gpu_capable = True
elif on_remote and host:
try:
# Activate the configured venv FIRST so the probe
# runs against the same python the launch script
# would activate. Without this prefix, bare
# `python3` was checked — which can disagree with
# the venv's wheel (e.g. user-site has CUDA wheel
# but venv has CPU-only), and the dep panel then
# showed "ready" green while every launch fell to
# CPU.
_vp = _venv_activate_prefix(venv)
probe = (
f'{_vp}python3 -c "import llama_cpp; import sys; '
'sys.exit(0 if llama_cpp.llama_supports_gpu_offload() else 1)" '
'&& echo llama_cpp_gpu=1 || echo llama_cpp_gpu=0; '
'command -v nvidia-smi >/dev/null 2>&1 '
'&& nvidia-smi -L 2>/dev/null | grep -q "GPU " '
'&& echo nvidia=1 || echo nvidia=0'
)
argv = _ssh_base_argv(host, ssh_port) + [probe]
proc = await asyncio.create_subprocess_exec(
*argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
)
out, _ = await asyncio.wait_for(proc.communicate(), timeout=8)
txt = out.decode("utf-8", errors="replace")
if "llama_cpp_gpu=1" in txt:
_gpu_capable = True
if "nvidia=1" in txt:
_has_nvidia_target = True
except Exception:
pass
else:
try:
import llama_cpp as _lcp # type: ignore
_gpu_capable = bool(_lcp.llama_supports_gpu_offload())
except Exception:
_gpu_capable = False
_has_nvidia_target = shutil.which("nvidia-smi") is not None
if (not _gpu_capable) and _has_nvidia_target:
pkg["partial"] = True
pkg["partial_reason"] = "Installed but CPU-only wheel — GPU detected on this target. Upgrade to a CUDA wheel for ~10× faster inference."
pkg["partial_action"] = "reinstall_llama_cpp_cuda"
# Attach per-package system_prereqs status. We probed each
# prereq name above; surface "Missing build deps: …" ONLY
# when the package itself is not installed — if the package
# works (e.g. llama-cpp-python already imports cleanly), the
# build toolchain is irrelevant and surfacing it as a red
# flag confuses users ("ready" + "missing" on the same row).
_prereqs = list(pkg.get("system_prereqs") or [])
if _prereqs:
if on_remote:
_pr_present = {n: bool(remote_status.get(n)) for n in _prereqs}
else:
_pr_present = {n: shutil.which(n) is not None for n in _prereqs}
pkg["system_prereqs_status"] = _pr_present
_missing = [n for n, ok in _pr_present.items() if not ok]
# Suppress the "missing build deps" hint when the package
# itself is installed — build deps are only relevant if
# the user would need to recompile from source.
if pkg.get("installed"):
_missing = []
if _missing:
# Build a target-specific install command from the
# (os_family, backend) matrix when we know both. Fall
# back to the multi-distro hint only when the target's
# OS can't be classified (e.g. ssh probe failed).
_resolved_os = target_os_id or "debian" # safest default
_cmd = _install_cmd_for_target(_resolved_os, backend or "", _missing)
if _cmd and target_os_id:
_hint = "Missing build deps for this target: " + ", ".join(_missing)
pkg["install_cmd_for_target"] = _cmd
pkg["install_cmd_os"] = target_os_id
pkg["install_cmd_backend"] = (backend or "").lower()
else:
_hint = "Missing build deps: " + ", ".join(_missing) + ". Install via apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git."
_existing_note = pkg.get("status_note") or ""
pkg["status_note"] = (_existing_note + " — " + _hint) if _existing_note else _hint
pkg["build_deps_missing"] = _missing
if pkg.get("installed"):
update_status = _package_pip_update_status(pkg, probe)
pkg["pip_update_available"] = update_status.available
@@ -1562,9 +1241,6 @@ def setup_shell_routes() -> APIRouter:
in_container=_running_in_container() if not on_remote else False,
installed=pkg["installed"],
default_hint=pkg.get("install_hint"),
host_docker_access=(
_host_docker_access_enabled() if not on_remote else False
),
)
pkg["applicable"] = status.applicable
pkg["install_hint"] = status.install_hint
@@ -1600,7 +1276,6 @@ def setup_shell_routes() -> APIRouter:
"onnxruntime",
"hdbscan",
"vllm",
"mlx-lm",
}
if pip_name not in known:
return {"ok": False, "error": f"Unknown package: {pip_name}"}
@@ -1613,127 +1288,6 @@ def setup_shell_routes() -> APIRouter:
return {"ok": True, "output": stdout.decode()[-200:]}
return {"ok": False, "error": stderr.decode()[-300:]}
@router.post("/api/cookbook/install-system-deps")
async def install_system_deps(request: Request):
"""Install OS-level system packages (cmake/build-essential/git/tmux)
on a remote target or in the local container. Admin only.
Bounded by a per-package allowlist — anything outside the catalog
is rejected so the route can't be coerced into installing arbitrary
OS packages. Uses `sudo -n` (passwordless) so the call returns a
clear "needs sudo password" error instead of hanging when interactive
sudo is required.
"""
_require_admin(request)
body = await request.json()
raw = body.get("packages") or []
host = (body.get("remote_host") or "").strip()
ssh_port = body.get("ssh_port")
# Names users can request — must match canonical names used in the
# deps catalog's `system_prereqs` field and on the System rows.
ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make"}
pkgs = [str(p).strip() for p in raw if str(p).strip() in ALLOWED]
if not pkgs:
return {"ok": False, "error": "no installable packages requested (allowlist: " + ", ".join(sorted(ALLOWED)) + ")"}
# Re-map to the right package name per OS. apt/dpkg use the names
# as-is; pacman has base-devel for build-essential, etc.
def _apt(names): return list(names)
def _pacman(names):
return ["base-devel" if n == "build-essential" else n for n in names]
def _dnf(names):
out = []
for n in names:
if n == "build-essential": out += ["gcc", "gcc-c++", "make"]
elif n == "g++": out += ["gcc-c++"]
else: out.append(n)
return out
def _apk(names):
out = []
for n in names:
if n == "build-essential": out.append("build-base")
else: out.append(n)
return out
def _zypper(names):
out = []
for n in names:
if n == "build-essential": out += ["gcc-c++", "make"]
elif n == "g++": out.append("gcc-c++")
else: out.append(n)
return out
def _brew(names):
return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
# Build a single shell snippet that detects the package manager and
# runs the right install. Non-interactive sudo (-n) only — if sudo
# asks for a password the script reports it instead of hanging.
apt_pkgs = " ".join(shlex.quote(p) for p in _apt(pkgs))
pac_pkgs = " ".join(shlex.quote(p) for p in _pacman(pkgs))
dnf_pkgs = " ".join(shlex.quote(p) for p in _dnf(pkgs))
apk_pkgs = " ".join(shlex.quote(p) for p in _apk(pkgs))
zypper_pkgs = " ".join(shlex.quote(p) for p in _zypper(pkgs))
brew_pkgs = " ".join(shlex.quote(p) for p in _brew(pkgs))
# Error messages go to stderr (>&2) so the route's error field
# gets populated. Without the redirect, `echo "ERROR…"` on stdout
# left stderr empty and the frontend toast fell through to a
# bare "HTTP 200" instead of surfacing the real reason.
script = (
'set -e; '
'BREW="$(command -v brew 2>/dev/null || true)"; '
'if [ -z "$BREW" ] && [ -x /opt/homebrew/bin/brew ]; then BREW=/opt/homebrew/bin/brew; fi; '
'if [ -z "$BREW" ] && [ -x /usr/local/bin/brew ]; then BREW=/usr/local/bin/brew; fi; '
'if [ -n "$BREW" ]; then '
f' if [ -z "{brew_pkgs}" ]; then echo "Nothing to install with brew for requested packages." >&2; exit 4; fi; "$BREW" install {brew_pkgs}; exit $?; '
'fi; '
'if [ "$(id -u)" = "0" ]; then SUDO=""; '
'elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then SUDO="sudo -n"; '
'else '
' echo "ERROR: this target needs sudo for its OS package manager, but passwordless sudo is unavailable. Open a terminal on the target and run the shown install command once, then retry in Cookbook." >&2; exit 2; fi; '
'if command -v apt-get >/dev/null 2>&1; then '
f' $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq && $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
'elif command -v pacman >/dev/null 2>&1; then '
f' $SUDO pacman -Sy --needed --noconfirm {pac_pkgs}; '
'elif command -v dnf >/dev/null 2>&1; then '
f' $SUDO dnf install -y {dnf_pkgs}; '
'elif command -v apk >/dev/null 2>&1; then '
f' $SUDO apk add --no-interactive {apk_pkgs}; '
'elif command -v zypper >/dev/null 2>&1; then '
f' $SUDO zypper --non-interactive install {zypper_pkgs}; '
'else '
' echo "ERROR: no supported package manager (apt/pacman/dnf/apk/zypper/brew) on this target." >&2; exit 3; fi'
)
try:
if host:
argv = _ssh_base_argv(host, ssh_port) + [script]
else:
argv = ["bash", "-lc", script]
except ValueError as e:
raise HTTPException(400, str(e))
try:
proc = await asyncio.create_subprocess_exec(
*argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
out, err = await asyncio.wait_for(proc.communicate(), timeout=180)
except asyncio.TimeoutError:
return {"ok": False, "error": "Install timed out after 180s"}
ok = (proc.returncode == 0)
# Combine stderr + (last lines of stdout) into a single error
# blob when ok=False — some package managers print useful failure
# context to stdout, and a script that exits via `echo ...; exit N`
# without `>&2` would otherwise hand back an empty error string
# and force the frontend to show a bare "HTTP 200".
err_txt = err.decode("utf-8", errors="replace").strip()
out_txt = out.decode("utf-8", errors="replace").strip()
if not ok:
tail_out = out_txt[-500:] if out_txt else ""
combined = err_txt or tail_out or f"exit code {proc.returncode}"
else:
combined = None
return {
"ok": ok,
"exit_code": proc.returncode,
"output": out_txt[-1000:],
"error": combined,
}
@router.post("/api/cookbook/rebuild-engine")
async def rebuild_engine(request: Request):
"""Clear the cached llama.cpp build so the next serve recompiles.
@@ -1754,8 +1308,7 @@ def setup_shell_routes() -> APIRouter:
return {"ok": False, "error": f"Unsupported engine: {engine}"}
host = str(body.get("remote_host") or "").strip()
ssh_port = body.get("ssh_port")
update_source = bool(body.get("update_source"))
cmd = _llama_cpp_rebuild_cmd(update_source=update_source)
cmd = _llama_cpp_rebuild_cmd()
try:
argv = (
(_ssh_base_argv(host, ssh_port) + [cmd])
routes/skills_routes.py
+1 -11
View File
@@ -22,16 +22,6 @@ from core.middleware import require_admin
logger = logging.getLogger(__name__)
# Last-resort verdict extraction from a teacher/verifier model's prose (run when
# JSON parsing fails). `["\'\s:]*` already consumes whitespace, so the original
# trailing `\s*` made two adjacent \s-matching quantifiers that backtrack O(n^2)
# on a `verdict` + whitespace flood in untrusted model output (CodeQL
# py/polynomial-redos). Without it a single unbounded quantifier remains — the
# matched text is identical, and the scan is linear.
_VERDICT_PROSE_RE = re.compile(
r'verdict["\'\s:]*["\']?(pass|needs_work|fail|inconclusive)', re.I
)
class SkillAddRequest(BaseModel):
# New schema (preferred)
@@ -206,7 +196,7 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
# Last resort: pull the verdict keyword straight out of the prose so a
# clearly-decided run isn't thrown away as "unparseable".
if v not in _VERDICTS:
km = _VERDICT_PROSE_RE.search(text)
km = _re.search(r'verdict["\'\s:]*\s*["\']?(pass|needs_work|fail|inconclusive)', text, _re.I)
if km:
v = km.group(1).lower()
if data is None:
routes/task_routes.py
+25 -42
View File
@@ -14,11 +14,6 @@ from core.database import SessionLocal, ScheduledTask, TaskRun
from core.constants import internal_api_base
from src.auth_helpers import get_current_user
from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
from src.task_action_policy import (
ADMIN_ONLY_TASK_ACTIONS,
is_admin_only_task_action,
owner_has_admin_task_privileges,
)
from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS
from routes.prefs_routes import _load_for_user, _save_for_user
@@ -421,18 +416,28 @@ def setup_task_routes(task_scheduler) -> APIRouter:
db.close()
return {"ok": True, "opened": True, "enabled": bool(prefs.get("tasks_enabled")), "resumed": resumed}
# Actions that execute shell/SSH commands or cross into admin-only
# Cookbook serving surfaces — restricted to admins.
# Actions that execute shell/SSH commands — restricted to admins.
# Non-admin users cannot create tasks with these action types via the
# API. See review CRIT-C.
_ADMIN_ONLY_ACTIONS = ADMIN_ONLY_TASK_ACTIONS
_ADMIN_ONLY_ACTIONS = {"run_local", "run_script", "ssh_command"}
def _is_admin(user: str | None) -> bool:
return owner_has_admin_task_privileges(user)
def _require_admin_for_task_action(user: str | None, task_type: str | None, action: str | None) -> None:
if is_admin_only_task_action(task_type, action) and not _is_admin(user):
raise HTTPException(403, f"Action '{action}' requires admin privileges")
if not user:
return False
# In-process tool-loopback marker — AuthMiddleware validated
# the internal token + loopback client before stamping this,
# so treat as admin-equivalent.
if user == "internal-tool":
return True
try:
from core.auth import AuthManager
auth = AuthManager()
if not auth.is_configured:
# Unconfigured single-user deploy: trust the local owner.
return True
return bool(auth.is_admin(user))
except Exception:
return False
def _validate_then_task_id(db, then_task_id: Optional[str], user: Optional[str], current_task_id: Optional[str] = None) -> Optional[str]:
target_id = (then_task_id or "").strip()
@@ -460,7 +465,8 @@ def setup_task_routes(task_scheduler) -> APIRouter:
# Block shell-executing action types for non-admins. action_run_local
# uses subprocess.run(shell=True) and ssh_command / run_script run
# arbitrary commands.
_require_admin_for_task_action(user, req.task_type, req.action)
if req.task_type == "action" and req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
if req.trigger_type == "schedule" and not req.schedule:
raise HTTPException(400, "Schedule is required for schedule-triggered tasks")
if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression:
@@ -587,7 +593,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
cache_tables = {
"summarize_emails": ("email_summaries",),
"draft_email_replies": ("email_ai_replies",),
"email_auto_translate": ("email_translations",),
"extract_email_events": ("email_calendar_extractions",),
"learn_sender_signatures": ("sender_signatures",),
"check_email_urgency": ("email_tags", "email_urgency_alerts"),
@@ -674,10 +679,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
if user and task.owner != user:
raise HTTPException(403, "Access denied")
next_task_type = req.task_type if req.task_type is not None else task.task_type
next_action = req.action if req.action is not None else task.action
_require_admin_for_task_action(user, next_task_type, next_action)
if req.name is not None:
task.name = req.name
if req.prompt is not None:
@@ -685,6 +686,9 @@ def setup_task_routes(task_scheduler) -> APIRouter:
if req.task_type is not None:
task.task_type = req.task_type
if req.action is not None:
# Same admin-only gate as create — see CRIT-C.
if req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
task.action = req.action
if req.output_target is not None:
task.output_target = req.output_target
@@ -801,7 +805,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
raise HTTPException(404, "Task not found")
if user and task.owner != user:
raise HTTPException(403, "Access denied")
_require_admin_for_task_action(user, task.task_type, task.action)
task.status = "active"
if (task.trigger_type or "schedule") == "schedule":
task.next_run = compute_next_run(
@@ -864,7 +867,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
raise HTTPException(404, "Task not found")
if user and task.owner != user:
raise HTTPException(403, "Access denied")
_require_admin_for_task_action(user, task.task_type, task.action)
finally:
db.close()
started = await task_scheduler.run_task_now(task_id, force=force)
@@ -890,11 +892,10 @@ def setup_task_routes(task_scheduler) -> APIRouter:
return {"ok": True, "message": "Task stopped"}
@router.get("/runs/recent")
async def list_recent_runs(request: Request, limit: int = 50, max_result_chars: int = 6000):
async def list_recent_runs(request: Request, limit: int = 50):
"""Recent task runs across ALL tasks for this owner. Drives the Activity view."""
user = _owner(request)
limit = max(1, min(limit, 200))
max_result_chars = max(500, min(max_result_chars, 20000))
db = SessionLocal()
try:
q = db.query(TaskRun, ScheduledTask).join(
@@ -928,20 +929,10 @@ def setup_task_routes(task_scheduler) -> APIRouter:
deduped.append((r, t))
if len(deduped) >= limit:
break
def _clip_run(r: TaskRun) -> dict:
d = _run_to_dict(r)
for key in ("result", "error"):
val = d.get(key)
if isinstance(val, str) and len(val) > max_result_chars:
d[key] = val[:max_result_chars].rstrip() + "\n\n[Activity preview truncated]"
return d
return {
"has_more": len(rows) > len(deduped),
"runs": [
{
**_clip_run(r),
**_run_to_dict(r),
"task_name": _display_task_name(t),
"task_type": t.task_type or "llm",
"action": t.action,
@@ -1054,14 +1045,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
).first()
if not task:
raise HTTPException(404, "Not found")
if (
is_admin_only_task_action(task.task_type, task.action)
and not owner_has_admin_task_privileges(task.owner)
):
task.status = "paused"
task.next_run = None
db.commit()
raise HTTPException(403, f"Action '{task.action}' requires admin privileges")
finally:
db.close()
started = await task_scheduler.run_task_now(task_id)
routes/upload_routes.py
+29 -290
View File
@@ -3,147 +3,18 @@ import os
import time
import json
import asyncio
import shutil
import uuid
from pathlib import Path
from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
from typing import List, Optional
from fastapi import APIRouter, Request, File, UploadFile, HTTPException
from typing import List
import logging
from core.middleware import require_admin
from core.database import (
SessionLocal,
ChatMessage as DbChatMessage,
CalendarCal,
CalendarEvent,
Document,
DocumentVersion,
GalleryImage,
Note,
Session as DbSession,
)
from src.auth_helpers import effective_user
from src.attachment_refs import attachment_refs_from_metadata
from src.constants import GENERATED_IMAGES_DIR
from src.upload_handler import (
UploadCleanupSafetyError,
count_recent_uploads,
extract_upload_ids,
)
from src.auth_helpers import get_current_user
from src.upload_handler import count_recent_uploads
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/upload", tags=["upload"])
UPLOAD_RESPONSE_HEADERS = {"X-Content-Type-Options": "nosniff"}
def _upload_ids_from_persisted_text(value: object) -> set[str]:
"""Return canonical upload IDs embedded in persisted text.
This covers attachment reference lines/URIs and the PDF source markers
stored by the document editor. False positives are intentionally
conservative: retaining an extra upload is safer than deleting referenced
bytes.
"""
return extract_upload_ids(value)
def _upload_ids_from_message_metadata(raw_metadata: object) -> set[str]:
"""Extract attachment IDs from a persisted chat metadata JSON value.
Malformed metadata raises instead of being treated as an empty reference
set. The admin cleanup route catches that failure and aborts cleanup.
"""
if raw_metadata in (None, ""):
return set()
if isinstance(raw_metadata, str):
metadata = json.loads(raw_metadata)
else:
metadata = raw_metadata
if not isinstance(metadata, dict):
raise ValueError("chat message metadata must be a JSON object")
attachments = metadata.get("attachments")
if attachments is not None:
if not isinstance(attachments, list) or any(
not isinstance(item, dict) for item in attachments
):
raise ValueError("chat message attachments metadata is malformed")
ids = {
str(ref["attachment_id"])
for ref in attachment_refs_from_metadata(metadata)
if ref.get("attachment_id")
}
# Preserve canonical IDs even in older metadata shapes not normalized by
# attachment_refs_from_metadata().
ids.update(_upload_ids_from_persisted_text(json.dumps(metadata)))
return ids
def _collect_persisted_upload_references() -> tuple[set[str], set[str]]:
"""Collect upload IDs/hashes still referenced by durable application data.
The caller must treat any exception as an incomplete scan and fail closed.
There is no distinct artifact table in the current schema; artifact-like
attachment references persisted in chat/document text are covered by the
canonical-ID scan.
"""
referenced_ids: set[str] = set()
referenced_hashes: set[str] = set()
db = SessionLocal()
try:
for content, raw_metadata in db.query(
DbChatMessage.content,
DbChatMessage.meta_data,
).yield_per(500):
referenced_ids.update(_upload_ids_from_persisted_text(content))
referenced_ids.update(_upload_ids_from_message_metadata(raw_metadata))
for (content,) in db.query(Document.current_content).yield_per(500):
referenced_ids.update(_upload_ids_from_persisted_text(content))
for (content,) in db.query(DocumentVersion.content).yield_per(500):
referenced_ids.update(_upload_ids_from_persisted_text(content))
for filename, file_hash in db.query(
GalleryImage.filename,
GalleryImage.file_hash,
).yield_per(500):
referenced_ids.update(_upload_ids_from_persisted_text(filename))
if file_hash:
referenced_hashes.add(str(file_hash))
for image_url, color, content, items in db.query(
Note.image_url,
Note.color,
Note.content,
Note.items,
).yield_per(500):
for value in (image_url, color, content, items):
referenced_ids.update(_upload_ids_from_persisted_text(value))
for (color,) in db.query(CalendarCal.color).yield_per(500):
referenced_ids.update(_upload_ids_from_persisted_text(color))
for color, description, location in db.query(
CalendarEvent.color,
CalendarEvent.description,
CalendarEvent.location,
).yield_per(500):
for value in (color, description, location):
referenced_ids.update(_upload_ids_from_persisted_text(value))
return referenced_ids, referenced_hashes
finally:
db.close()
def _run_reference_safe_cleanup(upload_handler) -> int:
referenced_ids, referenced_hashes = _collect_persisted_upload_references()
return upload_handler.cleanup_old_uploads(
referenced_upload_ids=referenced_ids,
referenced_upload_hashes=referenced_hashes,
)
def setup_upload_routes(upload_handler):
"""Setup upload routes with the provided handler"""
@@ -180,89 +51,9 @@ def setup_upload_routes(upload_handler):
raise HTTPException(404, "File not found")
def _valid_session_id_for_owner(db, session_id: str | None, owner: str | None) -> str | None:
if not session_id:
return None
sess = db.query(DbSession).filter(DbSession.id == session_id).first()
if not sess:
return None
if owner and sess.owner and sess.owner != owner:
return None
return session_id
def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
"""Make chat-uploaded images visible in Gallery without changing chat storage."""
is_image_file = getattr(upload_handler, "is_image_file", None)
if not callable(is_image_file):
return None
if not is_image_file(meta.get("name", ""), meta.get("mime", "")):
return None
source_path = meta.get("path")
if not source_path or not os.path.isfile(source_path):
return None
db = SessionLocal()
try:
file_hash = meta.get("hash")
if file_hash:
q = db.query(GalleryImage).filter(
GalleryImage.file_hash == file_hash,
GalleryImage.is_active == True, # noqa: E712
)
if owner:
q = q.filter(GalleryImage.owner == owner)
existing = q.first()
if existing:
return existing.id
image_dir = Path(GENERATED_IMAGES_DIR)
image_dir.mkdir(parents=True, exist_ok=True)
ext = Path(meta.get("name") or source_path).suffix.lower()
if ext not in {".png", ".jpg", ".jpeg", ".webp", ".gif"}:
mime_ext = {
"image/png": ".png",
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/webp": ".webp",
"image/gif": ".gif",
}.get(meta.get("mime", ""))
ext = mime_ext or ".png"
filename = f"{uuid.uuid4().hex[:12]}{ext}"
dest_path = image_dir / filename
shutil.copy2(source_path, dest_path)
image_id = str(uuid.uuid4())
db.add(GalleryImage(
id=image_id,
filename=filename,
prompt=meta.get("name") or "Chat upload",
model="chat-upload",
owner=owner,
session_id=_valid_session_id_for_owner(db, session_id, owner),
file_hash=file_hash,
width=meta.get("width"),
height=meta.get("height"),
file_size=meta.get("size"),
))
db.commit()
return image_id
except Exception as e:
db.rollback()
logger.warning("Failed to add chat image upload to gallery: %s", e)
return None
finally:
db.close()
@router.post("")
async def api_upload(
request: Request,
files: List[UploadFile] = File(...),
session_id: Optional[str] = Form(None),
):
async def api_upload(request: Request, files: List[UploadFile] = File(...)):
"""Upload files with enhanced security and organization."""
if not isinstance(session_id, str):
session_id = None
if not files:
raise HTTPException(400, "No files uploaded")
@@ -287,25 +78,18 @@ def setup_upload_routes(upload_handler):
for u in files:
try:
owner = effective_user(request)
meta = upload_handler.save_upload(u, client_ip, owner=owner)
gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
item = {
meta = upload_handler.save_upload(u, client_ip, owner=get_current_user(request))
out.append({
"id": meta["id"],
"name": meta["name"],
"mime": meta["mime"],
"size": meta["size"],
"hash": meta["hash"],
"checksum_sha256": meta.get("checksum_sha256") or meta["hash"],
"uploaded_at": meta["uploaded_at"],
"created_at": meta.get("created_at") or meta["uploaded_at"],
"width": meta.get("width"),
"height": meta.get("height"),
"is_duplicate": meta.get("is_duplicate", False)
}
if gallery_id:
item["gallery_id"] = gallery_id
out.append(item)
})
except HTTPException:
raise
except Exception as e:
@@ -321,23 +105,7 @@ def setup_upload_routes(upload_handler):
async def manual_cleanup(request: Request):
"""Manually trigger cleanup of old uploads."""
require_admin(request)
try:
cleaned_count = await asyncio.to_thread(
_run_reference_safe_cleanup,
upload_handler,
)
except UploadCleanupSafetyError:
logger.exception("Upload cleanup aborted because index safety checks failed")
raise HTTPException(
503,
"Upload cleanup aborted because upload index integrity could not be verified",
)
except Exception:
logger.exception("Upload cleanup skipped because reference discovery failed")
raise HTTPException(
503,
"Upload cleanup skipped because persisted references could not be verified",
)
cleaned_count = upload_handler.cleanup_old_uploads()
return {"status": "success", "files_cleaned": cleaned_count}
@router.get("/stats")
@@ -360,16 +128,17 @@ def setup_upload_routes(upload_handler):
import mimetypes as _mt
# Look up original filename and owner from uploads.json
original_name = file_id
# _load_upload_index() tolerates a missing/corrupt uploads.json (it falls
# back to the .bak sibling, then to {}), so a truncated DB degrades to
# "no metadata" instead of a 500 from an unhandled JSONDecodeError.
db = upload_handler._load_upload_index()
info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
if info:
original_name = info.get("name", file_id)
info = None
uploads_db = os.path.join(_upload_root(), "uploads.json")
if os.path.exists(uploads_db):
with open(uploads_db, encoding="utf-8") as f:
db = json.load(f)
info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
if info:
original_name = info.get("name", file_id)
auth_mgr = getattr(request.app.state, "auth_manager", None)
auth_configured = bool(auth_mgr and auth_mgr.is_configured)
current_user = effective_user(request)
current_user = get_current_user(request)
file_owner = info.get("owner") if info else None
if auth_configured:
if not current_user:
@@ -412,42 +181,19 @@ def setup_upload_routes(upload_handler):
def _load_upload_info(file_id: str):
"""Look up the uploads.json record for a file_id, with owner/auth checks."""
# Corruption-tolerant load (see download_file): a bad uploads.json yields
# {} rather than raising JSONDecodeError out of the vision path.
db = upload_handler._load_upload_index()
return next((fi for fi in db.values() if fi.get("id") == file_id), None)
info = None
uploads_db = os.path.join(_upload_root(), "uploads.json")
if os.path.exists(uploads_db):
with open(uploads_db, encoding="utf-8") as f:
db = json.load(f)
info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
return info
def _vision_cache_path(file_id: str) -> str:
cache_dir = os.path.join(_upload_root(), ".vision")
os.makedirs(cache_dir, exist_ok=True)
return os.path.join(cache_dir, file_id + ".txt")
def _sync_gallery_caption_for_upload(info: dict | None, owner: str | None, text: str) -> None:
"""Copy upload OCR/vision text onto the promoted gallery image row."""
if not info:
return
file_hash = info.get("hash")
if not file_hash:
return
db = SessionLocal()
try:
q = db.query(GalleryImage).filter(
GalleryImage.file_hash == file_hash,
GalleryImage.is_active == True, # noqa: E712
)
if owner:
q = q.filter(GalleryImage.owner == owner)
img = q.first()
if not img:
return
img.caption = (text or "").strip()
db.commit()
except Exception as e:
db.rollback()
logger.warning("Failed to sync OCR caption to gallery image: %s", e)
finally:
db.close()
@router.get("/{file_id}/vision")
async def get_vision_text(request: Request, file_id: str, force: int = 0):
"""Return the vision-model OCR/description for an uploaded image.
@@ -458,7 +204,7 @@ def setup_upload_routes(upload_handler):
info = _load_upload_info(file_id)
auth_mgr = getattr(request.app.state, "auth_manager", None)
auth_configured = bool(auth_mgr and auth_mgr.is_configured)
current_user = effective_user(request)
current_user = get_current_user(request)
file_owner = info.get("owner") if info else None
if auth_configured:
if not current_user:
@@ -474,9 +220,7 @@ def setup_upload_routes(upload_handler):
if not force and os.path.exists(cache_path):
try:
with open(cache_path, encoding="utf-8") as f:
cached_text = f.read()
_sync_gallery_caption_for_upload(info, file_owner or current_user, cached_text)
return {"text": cached_text, "cached": True}
return {"text": f.read(), "cached": True}
except Exception as e:
logger.warning(f"Vision cache read failed for {file_id}: {e}")
from src.document_processor import analyze_image_with_vl
@@ -490,7 +234,6 @@ def setup_upload_routes(upload_handler):
f.write(text)
except Exception as e:
logger.warning(f"Vision cache write failed for {file_id}: {e}")
_sync_gallery_caption_for_upload(info, file_owner or current_user, text)
return {"text": text, "cached": False}
@router.put("/{file_id}/vision")
@@ -504,7 +247,7 @@ def setup_upload_routes(upload_handler):
raise HTTPException(404, "File not found")
auth_mgr = getattr(request.app.state, "auth_manager", None)
auth_configured = bool(auth_mgr and auth_mgr.is_configured)
current_user = effective_user(request)
current_user = get_current_user(request)
file_owner = info.get("owner")
if auth_configured:
if not current_user:
@@ -512,16 +255,12 @@ def setup_upload_routes(upload_handler):
if file_owner != current_user and not auth_mgr.is_admin(current_user):
raise HTTPException(404, "File not found")
_resolve_upload_path(file_id)
try:
body = await request.json()
except json.JSONDecodeError:
raise HTTPException(400, "Request body must be valid JSON")
body = await request.json()
text = (body or {}).get("text", "")
if not isinstance(text, str):
raise HTTPException(400, "text must be a string")
with open(_vision_cache_path(file_id), "w", encoding="utf-8") as f:
f.write(text)
_sync_gallery_caption_for_upload(info, file_owner or current_user, text)
return {"ok": True}
async def periodic_rate_limit_cleanup():
routes/webhook_routes.py
+5 -5
View File
@@ -1,5 +1,6 @@
"""Webhook, API Token, and sync chat routes."""
import asyncio
import uuid
import logging
from typing import Optional
@@ -345,9 +346,8 @@ def setup_webhook_routes(
resp = await client.get(models_url, headers=hdrs)
resp.raise_for_status()
data = resp.json()
items = data if isinstance(data, list) else (data.get("data") or [])
ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
if not ids and isinstance(data, dict):
ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
if not ids:
ids = [
m.get("name") or m.get("model")
for m in (data.get("models") or [])
@@ -385,10 +385,10 @@ def setup_webhook_routes(
sess.add_message(ChatMessage("assistant", reply))
session_manager.save_sessions()
webhook_manager.fire_and_forget("chat.completed", {
asyncio.create_task(webhook_manager.fire("chat.completed", {
"session_id": session_id, "model": sess.model,
"user_message": message[:2000], "response": reply[:2000],
})
}))
return {"response": reply, "session_id": session_id, "model": sess.model}
scripts/claim_ownerless.py
+2 -8
View File
@@ -27,18 +27,12 @@ def claim_json_entries(entries, owner):
return count
def owner_arg(argv):
if len(argv) < 2 or not argv[1].strip():
return None
return argv[1].strip()
def main():
owner = owner_arg(sys.argv)
if not owner:
if len(sys.argv) < 2:
print("Usage: python scripts/claim_ownerless.py <username>")
sys.exit(1)
owner = sys.argv[1]
print(f"Claiming all ownerless data for: {owner}\n")
# 1. Memories (JSON files)
scripts/odysseus-backup
+9 -23
View File
@@ -133,30 +133,16 @@ def cmd_list(args):
emit([], args)
return
entries = []
for p in _BACKUP_DIR.iterdir():
entry = _backup_entry(p)
if entry is not None:
entries.append(entry)
entries.sort(key=lambda entry: entry["_mtime"], reverse=True)
for entry in entries:
entry.pop("_mtime", None)
emit(entries, args)
def _backup_entry(p):
try:
for p in sorted(_BACKUP_DIR.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True):
if not p.is_file():
return None
st = p.stat()
except OSError:
return None
return {
"path": str(p),
"name": p.name,
"bytes": st.st_size,
"modified": datetime.fromtimestamp(st.st_mtime).isoformat(),
"_mtime": st.st_mtime,
}
continue
entries.append({
"path": str(p),
"name": p.name,
"bytes": p.stat().st_size,
"modified": datetime.fromtimestamp(p.stat().st_mtime).isoformat(),
})
emit(entries, args)
def cmd_verify(args):
scripts/odysseus-calendar
+1 -5
View File
@@ -103,13 +103,9 @@ def cmd_list(args) -> None:
end = _parse_dt(args.end) if args.end else (start + timedelta(days=30))
db = SessionLocal()
try:
# Overlap semantics, matching the web route (routes/calendar_routes.py)
# and the recurring-expansion contract: an event is in the window when
# it starts before the window end AND ends after the window start. This
# includes multi-day / in-progress events that began before `start`.
q = db.query(CalendarEvent).filter(
CalendarEvent.dtstart >= start,
CalendarEvent.dtstart < end,
CalendarEvent.dtend > start,
)
if args.calendar:
cal = db.query(CalendarCal).filter(CalendarCal.name == args.calendar).first()
scripts/odysseus-gallery
+11 -15
View File
@@ -38,27 +38,23 @@ def _preview_text(value, limit: int = 200) -> str:
return text[:limit]
def _text_field(value) -> str:
return value if isinstance(value, str) else ""
def _serialize_image(i: "GalleryImage") -> dict:
return {
"id": i.id,
"filename": _text_field(i.filename),
"filename": i.filename,
"prompt": _preview_text(i.prompt),
"model": _text_field(i.model),
"size": _text_field(i.size),
"tags": _text_field(i.tags),
"model": i.model or "",
"size": i.size or "",
"tags": i.tags or "",
"favorite": bool(i.favorite),
"album_id": _text_field(i.album_id),
"session_id": _text_field(i.session_id),
"album_id": i.album_id or "",
"session_id": i.session_id or "",
"width": i.width,
"height": i.height,
"file_size": i.file_size,
"taken_at": i.taken_at.isoformat() if i.taken_at else "",
"camera_make": _text_field(i.camera_make),
"camera_model": _text_field(i.camera_model),
"camera_make": i.camera_make or "",
"camera_model": i.camera_model or "",
"created_at": i.created_at.isoformat() if i.created_at else "",
}
@@ -97,11 +93,11 @@ def cmd_show(args):
if not i:
fail(f"no image with id {args.id!r}")
out = _serialize_image(i)
out["prompt_full"] = _text_field(i.prompt)
out["ai_tags"] = _text_field(i.ai_tags)
out["prompt_full"] = i.prompt or ""
out["ai_tags"] = i.ai_tags or ""
out["gps_lat"] = i.gps_lat or ""
out["gps_lng"] = i.gps_lng or ""
out["file_hash"] = _text_field(i.file_hash)
out["file_hash"] = i.file_hash or ""
emit(out, args)
finally:
db.close()
scripts/odysseus-mail
-2
View File
@@ -108,8 +108,6 @@ def _q(name: str) -> str:
def _split_recipients(value: str) -> list[str]:
if not isinstance(value, str):
return []
return [r.strip() for r in (value or "").split(",") if r.strip()]
scripts/odysseus-memory
+1 -1
View File
@@ -90,7 +90,7 @@ def cmd_add(args):
# add_entry doesn't save by default — the call in chat does it
# after dedup checks. Persist here so a one-shot CLI add sticks.
all_entries = _manager().load_all()
if not any(isinstance(e, dict) and e.get("id") == entry.get("id") for e in all_entries):
if not any(e.get("id") == entry.get("id") for e in all_entries):
all_entries.append(entry)
_manager().save(all_entries)
emit(entry, args)
scripts/odysseus-notes
+1 -3
View File
@@ -36,9 +36,7 @@ def _load_items(raw) -> list:
items = json.loads(raw)
except (TypeError, json.JSONDecodeError):
return []
if not isinstance(items, list):
return []
return [item for item in items if isinstance(item, dict)]
return items if isinstance(items, list) else []
def _serialize(n: "Note") -> dict:
services/hwfit/data/hf_models.json
+19 -349
View File
@@ -5113,8 +5113,8 @@
{
"name": "deepseek-ai/DeepSeek-V4-Flash",
"provider": "deepseek-ai",
"parameter_count": "158.1B",
"parameters_raw": 158069433298,
"parameter_count": "284B",
"parameters_raw": 284000000000,
"active_parameters": 13000000000,
"is_moe": true,
"min_ram_gb": 200.0,
@@ -5130,40 +5130,15 @@
],
"pipeline_tag": "text-generation",
"architecture": "deepseek_v4_moe",
"hf_downloads": 1882337,
"hf_likes": 1651,
"release_date": "2026-06-22"
},
{
"name": "deepseek-ai/DeepSeek-V4-Flash-DSpark",
"provider": "deepseek-ai",
"parameter_count": "165.3B",
"parameters_raw": 165265454782,
"active_parameters": 13000000000,
"is_moe": true,
"active_experts": 6,
"min_ram_gb": 170.0,
"recommended_ram_gb": 250.0,
"min_vram_gb": 165.0,
"quantization": "FP8-Mixed",
"context_length": 1000000,
"use_case": "General-purpose reasoning, long-context",
"capabilities": [
"long_context",
"reasoning",
"moe"
],
"pipeline_tag": "text-generation",
"architecture": "deepseek_v4_moe",
"hf_downloads": 4446,
"hf_likes": 107,
"release_date": "2026-06-27"
"hf_downloads": 3542202,
"hf_likes": 0,
"release_date": "2026-05-15"
},
{
"name": "deepseek-ai/DeepSeek-V4-Flash-Base",
"provider": "deepseek-ai",
"parameter_count": "292.0B",
"parameters_raw": 292021347282,
"parameter_count": "284B",
"parameters_raw": 284000000000,
"active_parameters": 13000000000,
"is_moe": true,
"min_ram_gb": 290.0,
@@ -5178,15 +5153,15 @@
],
"pipeline_tag": "text-generation",
"architecture": "deepseek_v4_moe",
"hf_downloads": 76030,
"hf_likes": 256,
"release_date": "2026-04-27"
"hf_downloads": 0,
"hf_likes": 0,
"release_date": "2026-05-15"
},
{
"name": "deepseek-ai/DeepSeek-V4-Pro",
"provider": "deepseek-ai",
"parameter_count": "861.6B",
"parameters_raw": 861608274846,
"parameter_count": "1.6T",
"parameters_raw": 1600000000000,
"active_parameters": 49000000000,
"is_moe": true,
"min_ram_gb": 1100.0,
@@ -5202,40 +5177,15 @@
],
"pipeline_tag": "text-generation",
"architecture": "deepseek_v4_moe",
"hf_downloads": 1154610,
"hf_likes": 5118,
"release_date": "2026-06-22"
},
{
"name": "deepseek-ai/DeepSeek-V4-Pro-DSpark",
"provider": "deepseek-ai",
"parameter_count": "889.5B",
"parameters_raw": 889484881098,
"active_parameters": 49000000000,
"is_moe": true,
"active_experts": 6,
"min_ram_gb": 900.0,
"recommended_ram_gb": 1250.0,
"min_vram_gb": 890.0,
"quantization": "FP8-Mixed",
"context_length": 1000000,
"use_case": "Flagship reasoning, long-context",
"capabilities": [
"long_context",
"reasoning",
"moe"
],
"pipeline_tag": "text-generation",
"architecture": "deepseek_v4_moe",
"hf_downloads": 6939,
"hf_likes": 241,
"release_date": "2026-06-27"
"hf_downloads": 0,
"hf_likes": 0,
"release_date": "2026-05-15"
},
{
"name": "deepseek-ai/DeepSeek-V4-Pro-Base",
"provider": "deepseek-ai",
"parameter_count": "1.6T",
"parameters_raw": 1600790440862,
"parameters_raw": 1600000000000,
"active_parameters": 49000000000,
"is_moe": true,
"min_ram_gb": 1700.0,
@@ -5250,9 +5200,9 @@
],
"pipeline_tag": "text-generation",
"architecture": "deepseek_v4_moe",
"hf_downloads": 25387,
"hf_likes": 305,
"release_date": "2026-04-27"
"hf_downloads": 0,
"hf_likes": 0,
"release_date": "2026-05-15"
},
{
"name": "deepseek-ai/deepseek-coder-6.7b-base",
@@ -13358,106 +13308,6 @@
"_discovered": true,
"gguf_sources": []
},
{
"name": "zai-org/GLM-5.2",
"provider": "zai-org",
"parameter_count": "753.3B",
"parameters_raw": 753329940480,
"min_ram_gb": 1510.0,
"recommended_ram_gb": 1800.0,
"min_vram_gb": 1510.0,
"quantization": "BF16",
"context_length": 1048576,
"use_case": "General purpose reasoning, coding, long-context",
"capabilities": [
"long_context",
"reasoning",
"coding",
"moe"
],
"pipeline_tag": "text-generation",
"architecture": "glm_moe_dsa",
"hf_downloads": 142547,
"hf_likes": 2996,
"release_date": "2026-06-23",
"is_moe": true,
"active_experts": 8,
"gguf_sources": [
{
"repo": "unsloth/GLM-5.2-GGUF",
"provider": "unsloth",
"file": "UD-Q4_K_M/*.gguf",
"quant": "Q4_K_M"
}
]
},
{
"name": "zai-org/GLM-5.2-FP8",
"provider": "zai-org",
"parameter_count": "753.4B",
"parameters_raw": 753375793584,
"min_ram_gb": 760.0,
"recommended_ram_gb": 900.0,
"min_vram_gb": 760.0,
"quantization": "FP8",
"context_length": 1048576,
"use_case": "General purpose reasoning, coding, long-context",
"capabilities": [
"long_context",
"reasoning",
"coding",
"moe"
],
"pipeline_tag": "text-generation",
"architecture": "glm_moe_dsa",
"hf_downloads": 884226,
"hf_likes": 182,
"release_date": "2026-06-23",
"is_moe": true,
"active_experts": 8,
"gguf_sources": [
{
"repo": "unsloth/GLM-5.2-GGUF",
"provider": "unsloth",
"file": "UD-Q4_K_M/*.gguf",
"quant": "Q4_K_M"
}
]
},
{
"name": "unsloth/GLM-5.2-GGUF",
"provider": "unsloth",
"parameter_count": "753.9B",
"parameters_raw": 753864139008,
"min_ram_gb": 452.0,
"recommended_ram_gb": 620.0,
"min_vram_gb": 452.0,
"quantization": "Q4_K_M",
"context_length": 1048576,
"use_case": "General purpose reasoning, coding, long-context (GGUF)",
"capabilities": [
"long_context",
"reasoning",
"coding",
"moe"
],
"pipeline_tag": "text-generation",
"architecture": "glm-dsa",
"hf_downloads": 180394,
"hf_likes": 474,
"release_date": "2026-06-23",
"is_moe": true,
"active_experts": 8,
"is_gguf": true,
"gguf_sources": [
{
"repo": "unsloth/GLM-5.2-GGUF",
"provider": "unsloth",
"file": "UD-Q4_K_M/*.gguf",
"quant": "Q4_K_M"
}
]
},
{
"name": "cyankiwi/Qwen3.5-35B-A3B-AWQ-4bit",
"provider": "cyankiwi",
@@ -14209,138 +14059,6 @@
"vision"
]
},
{
"name": "google/gemma-4-12B-it",
"provider": "Google",
"parameter_count": "12.0B",
"parameters_raw": 12000000000,
"min_ram_gb": 8.5,
"recommended_ram_gb": 11.0,
"min_vram_gb": 7.5,
"quantization": "Q4_K_M",
"context_length": 131072,
"use_case": "General purpose, multimodal; unsloth/gemma-4-12B-it-GGUF Dynamic variants reduce VRAM from ~7.5 GB to ~5.5 GB",
"is_moe": false,
"num_experts": null,
"active_experts": null,
"active_parameters": null,
"architecture": "gemma4",
"pipeline_tag": "image-text-to-text",
"release_date": "2026-04-01",
"gguf_sources": [
{
"repo": "unsloth/gemma-4-12B-it-GGUF",
"provider": "unsloth"
}
],
"capabilities": [
"vision"
]
},
{
"name": "google/gemma-4-12B-it-qat-int4",
"provider": "Google",
"parameter_count": "12.0B",
"parameters_raw": 12000000000,
"min_ram_gb": 8.0,
"recommended_ram_gb": 9.5,
"min_vram_gb": 6.5,
"quantization": "QAT-INT4",
"context_length": 131072,
"use_case": "General purpose, multimodal (QAT quantization-aware training — higher quality than post-train INT4; vLLM native; no GGUF)",
"is_moe": false,
"num_experts": null,
"active_experts": null,
"active_parameters": null,
"architecture": "gemma4",
"pipeline_tag": "image-text-to-text",
"release_date": "2026-04-01",
"gguf_sources": [],
"capabilities": [
"vision"
]
},
{
"name": "google/gemma-4-12B-it-qat-int8",
"provider": "Google",
"parameter_count": "12.0B",
"parameters_raw": 12000000000,
"min_ram_gb": 15.0,
"recommended_ram_gb": 20.0,
"min_vram_gb": 13.5,
"quantization": "QAT-INT8",
"context_length": 131072,
"use_case": "General purpose, multimodal (QAT INT8 — highest quality, 2x VRAM of QAT-INT4; vLLM native; no GGUF)",
"is_moe": false,
"num_experts": null,
"active_experts": null,
"active_parameters": null,
"architecture": "gemma4",
"pipeline_tag": "image-text-to-text",
"release_date": "2026-04-01",
"gguf_sources": [],
"capabilities": [
"vision"
]
},
{
"name": "google/gemma-4-12B-it-qat-q4_0-gguf",
"provider": "Google",
"parameter_count": "12.0B",
"parameters_raw": 12000000000,
"min_ram_gb": 8.5,
"recommended_ram_gb": 11.0,
"min_vram_gb": 7.5,
"quantization": "QAT-INT4",
"context_length": 262144,
"use_case": "General purpose, multimodal (vision + audio); official Google QAT int4 GGUF — near-bf16 quality at int4 size, served on llama.cpp/Ollama with CPU offload",
"is_moe": false,
"num_experts": null,
"active_experts": null,
"active_parameters": null,
"architecture": "gemma4",
"pipeline_tag": "image-text-to-text",
"release_date": "2026-04-01",
"gguf_sources": [
{
"repo": "google/gemma-4-12B-it-qat-q4_0-gguf",
"provider": "Google",
"file": "gemma-4-12b-it-qat-q4_0.gguf"
}
],
"capabilities": [
"vision",
"audio"
]
},
{
"name": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf",
"provider": "Google",
"parameter_count": "25.2B",
"parameters_raw": 25200000000,
"min_ram_gb": 14.4,
"recommended_ram_gb": 18.0,
"min_vram_gb": 14.4,
"quantization": "QAT-INT4",
"context_length": 262144,
"use_case": "High-throughput, multimodal MoE (3.8B active); official Google QAT int4 GGUF — near-bf16 quality at int4 size, served on llama.cpp with CPU offload",
"is_moe": true,
"num_experts": null,
"active_experts": null,
"active_parameters": 3800000000,
"architecture": "gemma4",
"pipeline_tag": "image-text-to-text",
"release_date": "2026-04-01",
"gguf_sources": [
{
"repo": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf",
"provider": "Google"
}
],
"capabilities": [
"vision"
]
},
{
"name": "google/gemma-4-31B-it",
"provider": "Google",
@@ -19105,54 +18823,6 @@
"active_experts": 8,
"active_parameters": 13600000000
},
{
"name": "MiniMaxAI/MiniMax-M3",
"provider": "MiniMaxAI",
"parameter_count": "427.0B",
"parameters_raw": 427040140160,
"min_ram_gb": 855.0,
"recommended_ram_gb": 1025.0,
"min_vram_gb": 855.0,
"quantization": "BF16",
"context_length": 1000000,
"use_case": "Vision, chat, coding, agentic tool use",
"capabilities": [
"vision",
"tool_use",
"coding",
"moe"
],
"pipeline_tag": "image-text-to-text",
"architecture": "minimax_m3_vl",
"hf_downloads": 192311,
"hf_likes": 1267,
"release_date": "2026-06-23",
"is_moe": true
},
{
"name": "MiniMaxAI/MiniMax-M3-MXFP8",
"provider": "MiniMaxAI",
"parameter_count": "440.3B",
"parameters_raw": 440279845760,
"min_ram_gb": 445.0,
"recommended_ram_gb": 560.0,
"min_vram_gb": 445.0,
"quantization": "MXFP8",
"context_length": 1000000,
"use_case": "Vision, chat, coding, agentic tool use",
"capabilities": [
"vision",
"tool_use",
"coding",
"moe"
],
"pipeline_tag": "image-text-to-text",
"architecture": "minimax_m3_vl",
"hf_downloads": 572278,
"hf_likes": 43,
"release_date": "2026-06-15",
"is_moe": true
},
{
"name": "bullerwins/MiniMax-M2.7-REAP-172B-fp8",
"provider": "bullerwins",
services/hwfit/data/mlx_community_models.json
-15727
View File
File diff suppressed because it is too large Load Diff
services/hwfit/fit.py
+21 -166
View File
@@ -9,7 +9,7 @@ from services.hwfit.models import (
GPU_BANDWIDTH = {
"5090": 1792, "5080": 960, "5070 ti": 896, "5070": 672, "5060 ti": 448, "5060": 256,
"4090": 1008, "4080 super": 736, "4080": 717, "4070 ti super": 672, "4070 ti": 504, "4070 super": 504, "4070": 504, "4060 ti": 288, "4060": 272,
"3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360, "3050 ti": 192, "3050": 224,
"3090 ti": 1008, "3090": 936, "3080 ti": 912, "3080": 760, "3070 ti": 608, "3070": 448, "3060 ti": 448, "3060": 360,
"2080 ti": 616, "2080 super": 496, "2080": 448, "2070 super": 448, "2070": 448, "2060 super": 448, "2060": 336,
"1660 ti": 288, "1660 super": 336, "1660": 192, "1650 super": 192, "1650": 128,
"h100 sxm": 3350, "h100": 2039, "h200": 4800, "a100 sxm": 2039, "a100": 1555,
@@ -19,10 +19,6 @@ GPU_BANDWIDTH = {
"6950 xt": 576, "6900 xt": 512, "6800 xt": 512, "6800": 512, "6700 xt": 384, "6600 xt": 256, "6600": 224,
"mi300x": 5300, "mi300": 5300, "mi250x": 3277, "mi250": 3277, "mi210": 1638, "mi100": 1229,
"9070 xt": 624, "9070": 488, "9060 xt": 322, "9060": 322,
# NVIDIA GB10 Grace-Blackwell superchip (DGX Spark). Unified LPDDR5X memory,
# not Apple Silicon, so it lives in the generic GPU table — the Apple-only
# lookup never matches it (its name carries no "apple").
"gb10": 273,
}
# Pre-sort keys by length descending for correct substring matching
@@ -130,57 +126,6 @@ def _lookup_bandwidth(system):
return None
def _canonical_cpu_backend(system):
"""Return the canonical CPU backend for cpu_only speed estimation.
Normalizes CPU-architecture aliases separately from the GPU backend, and
overrides GPU-only backends (CUDA/ROCm/Metal) so they do not inherit a
discrete-GPU fallback constant when the model is actually running on CPU.
"""
backend = (system.get("backend") or "").lower().strip()
cpu_arch = (system.get("cpu_arch") or "").lower().strip()
cpu_name = (system.get("cpu_name") or "").lower()
gpu_name = (system.get("gpu_name") or "").lower()
# Already-canonical CPU backends
if backend in ("cpu_x86", "cpu_arm"):
return backend
# Raw CPU-architecture aliases. Treat plain "arm" as 32-bit ARM, not the
# ARM64-class CPU fallback used for Apple Silicon/aarch64 machines.
if backend in ("x86_64", "amd64", "i386", "i686"):
return "cpu_x86"
if backend in ("arm64", "aarch64"):
return "cpu_arm"
# Prefer an explicit CPU architecture field when present
if cpu_arch:
if cpu_arch in ("x86_64", "amd64", "x86", "i386", "i686"):
return "cpu_x86"
if cpu_arch in ("arm64", "aarch64"):
return "cpu_arm"
# Apple Silicon enters ranking as backend="metal"; its CPU path is ARM.
if backend in ("metal", "mps", "apple") or "apple" in cpu_name or "apple" in gpu_name:
return "cpu_arm"
# Conservative default for CUDA/ROCm/discrete GPU backends and unknowns.
return "cpu_x86"
def _is_mlx_model(model, native_q=None):
name = (model.get("name") or "").lower()
provider = (model.get("provider") or "").lower()
fmt = (model.get("format") or "").lower()
q = (native_q if native_q is not None else _native_quant(model)).lower()
return (
q.startswith("mlx-")
or provider == "mlx-community"
or fmt == "mlx"
or name.startswith("mlx-community/")
)
def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
"""Estimate tok/s. Uses active params for MoE (only active experts run per token).
@@ -198,11 +143,6 @@ def _estimate_speed(model, quant, run_mode, system, offload_frac=0.0):
bw = _lookup_bandwidth(system)
backend = system.get("backend", "cpu_x86")
# CPU-only inference must never inherit a GPU backend's fallback constant,
# even if the detected system happens to report a CUDA/Metal/ROCm backend.
if run_mode == "cpu_only":
backend = _canonical_cpu_backend(system)
if bw and run_mode in ("gpu", "cpu_offload"):
bpp = QUANT_BYTES_PER_PARAM.get(quant, 0.5)
model_gb = pb * bpp
@@ -326,22 +266,6 @@ def _fit_score(required, available):
return 50
def _is_unified_memory_system(system):
backend = (system.get("backend") or "").lower()
return bool(system.get("unified_memory")) or backend in ("metal", "mps", "apple")
def _fit_level_for_budget(required_gb, budget_gb):
if not required_gb or not budget_gb or required_gb > budget_gb:
return "too_tight"
ratio = required_gb / budget_gb
if ratio <= 0.50:
return "perfect"
if ratio <= 0.78:
return "good"
return "marginal"
def _context_score(ctx, use_case):
target = CONTEXT_TARGET.get(use_case, 4096)
if ctx >= target:
@@ -545,42 +469,21 @@ def analyze_model(model, system, target_quant=None, scoring_use_case=None, targe
run_mode, quant, fit_ctx, required_gb = result
# Determine fit level
unified_memory = _is_unified_memory_system(system)
total_ram = system.get("total_ram_gb") or available_ram
unified_budget = max(total_ram or 0, available_ram or 0, effective_vram or 0)
budget = unified_budget if unified_memory else (effective_vram if run_mode == "gpu" else available_ram)
budget = effective_vram if run_mode == "gpu" else available_ram
if required_gb > budget:
return None
if run_mode == "gpu":
if unified_memory:
fit_level = _fit_level_for_budget(required_gb, budget)
else:
# GPU-only fit must leave real allocator/KV/runtime headroom. The
# old check used recommended_ram_gb (or required_gb as a fallback),
# which made any model that barely fit VRAM read as "perfect".
# On CUDA/vLLM/SGLang that is misleading: 141 GB on a 160 GB box is
# runnable, but not a comfortable perfect fit.
if gpu_vram >= required_gb * 1.50:
fit_level = "perfect"
elif gpu_vram >= required_gb * 1.2:
fit_level = "good"
else:
fit_level = "marginal"
elif run_mode == "cpu_offload":
fit_level = _fit_level_for_budget(required_gb, budget)
if fit_level == "perfect":
rec = model.get("recommended_ram_gb") or required_gb
if rec <= gpu_vram:
fit_level = "perfect"
elif gpu_vram >= required_gb * 1.2:
fit_level = "good"
else:
fit_level = _fit_level_for_budget(required_gb, budget)
if fit_level == "too_tight":
else:
fit_level = "marginal"
# Rows that comfortably fit in a huge RAM/unified-memory pool should not all
# look "marginal"; that made 1B-70B CPU/Ollama rows orange on 256 GB systems.
if fit_level == "marginal" and budget and required_gb <= budget * 0.78:
fit_level = "good"
if fit_level == "good" and budget and required_gb <= budget * 0.50 and run_mode != "cpu_offload":
fit_level = "perfect"
elif run_mode == "cpu_offload":
fit_level = "good" if available_ram >= required_gb * 1.2 else "marginal"
else:
fit_level = "marginal"
# Fraction of the model that spills to CPU RAM (drives the offload speed
# model). When offloading, anything beyond the GPU's VRAM lives in system RAM.
@@ -671,40 +574,6 @@ SORT_KEYS = {
}
def _search_blob(*parts):
text = " ".join(str(p or "") for p in parts).lower()
compact = re.sub(r"[^a-z0-9]+", "", text)
spaced = re.sub(r"[^a-z0-9]+", " ", text).strip()
return f"{text} {spaced} {compact}"
def _matches_search(model, search):
terms = [t for t in re.split(r"\s+", (search or "").strip().lower()) if t]
if not terms:
return True
blob = _search_blob(
model.get("name"),
model.get("provider"),
model.get("architecture"),
model.get("quantization"),
model.get("format"),
model.get("parameter_count"),
)
for term in terms:
norm = re.sub(r"[^a-z0-9]+", "", term)
if term not in blob and (not norm or norm not in blob):
if re.fullmatch(r"\d+(?:\.\d+)?b?", term):
try:
wanted = float(term.rstrip("b"))
actual = params_b(model)
except (TypeError, ValueError):
actual = 0
if wanted > 0 and actual > 0 and abs(actual - wanted) <= max(5.0, wanted * 0.08):
continue
return False
return True
def rank_models(system, use_case=None, limit=50, search=None, sort="score", quant=None, target_context=None, fit_only=False):
"""Rank all models against detected hardware. Returns sorted list of fit results.
@@ -777,11 +646,10 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
for m in models:
native_q = _native_quant(m)
is_mlx = _is_mlx_model(m, native_q)
# MLX is Apple Silicon-only. It should never appear on CUDA/ROCm/CPU,
# but it is first-class on Metal where mlx_lm.server can serve it.
if is_mlx and not apple_silicon:
# MLX needs the mlx_lm runtime, which Odysseus does not generate serve
# commands for. Hide it on every backend, including Metal.
if native_q.startswith("mlx-") or "mlx" in (m.get("name") or "").lower():
continue
# ROCm support for vLLM/SGLang quantized safetensors is too brittle to
@@ -808,7 +676,7 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
# Windows is the same: Odysseus only supports llama.cpp on Windows,
# which requires GGUF. vLLM/SGLang are explicitly blocked, so AWQ/GPTQ
# models without a GGUF source are unservable there.
if (apple_silicon or consumer_amd or is_windows) and not is_mlx and not (m.get("is_gguf") or m.get("gguf_sources")):
if (apple_silicon or consumer_amd or is_windows) and not (m.get("is_gguf") or m.get("gguf_sources")):
continue
# Format filter: AWQ tab -> only AWQ models, FP4 tab -> FP4-family models, etc.
@@ -826,26 +694,13 @@ def rank_models(system, use_case=None, limit=50, search=None, sort="score", quan
if quant in ("INT4", "INT8", "W4A16", "W8A8", "W8A16") and native_q != quant:
continue
if search and not _matches_search(m, search):
continue
if search:
name = m.get("name", "").lower()
provider = m.get("provider", "").lower()
if search.lower() not in name and search.lower() not in provider:
continue
model_quant = quant
# UI "Q4" means the user's looking for a 4-bit fit. On multi-GPU
# CUDA/vLLM/SGLang boxes, many practical 4-bit models are native AWQ
# safetensors, not GGUF Q4_K_M. If we pass Q4_K_M into a prequantized
# AWQ row, analyze_model correctly rejects it as the wrong serving
# format, but the result is confusing: highlighting Quant/Q4 hides the
# exact AWQ rows the machine is built to run. Treat Q4 as AWQ-4bit for
# native AWQ rows only on accelerator servers that can serve them.
if (
quant == "Q4_K_M"
and system.get("gpu_count", 1) >= 2
and not (apple_silicon or consumer_amd or is_windows)
and native_q == "AWQ-4bit"
):
model_quant = native_q
result = analyze_model(m, system, target_quant=model_quant, scoring_use_case=(use_case or "general"), target_context=target_context)
result = analyze_model(m, system, target_quant=quant, scoring_use_case=(use_case or "general"), target_context=target_context)
if result is None:
continue

Some files were not shown because too many files have changed in this diff Show More