From bfeea7f463eb089530b1a9ebb5f3bf7edc2cbb05 Mon Sep 17 00:00:00 2001 From: michaelxer Date: Sun, 28 Jun 2026 21:50:42 +0700 Subject: [PATCH 01/32] fix(docs): correct broken backup-restore link in setup.md Fixes #4926 - the link used docs/backup-restore.md from within docs/setup.md, which resolved to docs/docs/backup-restore.md (404). Changed to same-directory relative path. --- docs/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup.md b/docs/setup.md index 63a95f687..588ac3308 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -445,4 +445,4 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum `memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`. To back up or restore everything in `data/`, see the -[Backup & Restore guide](docs/backup-restore.md). +[Backup & Restore guide](backup-restore.md). From e157f1e63d955a96e25e8cfbf97ad4d093890be8 Mon Sep 17 00:00:00 2001 From: Mazen Tamer Salah <78306991+mazen-salah@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:01:57 +0300 Subject: [PATCH 02/32] fix(cookbook): stop Ollama runner from executing the install one-liner (#3926) The generated bash runner printed the missing-ollama hint with the install one-liner wrapped in backticks inside a double-quoted echo. Backticks in double quotes are command substitution, so on any serve target without ollama the script downloaded and ran the system-wide installer (including remote SSH hosts) instead of printing the hint. _validate_serve_cmd rejects backticks in user-supplied commands for exactly this reason; the app's own generated script never goes through that validator. Move the hint into OLLAMA_MISSING_HINT in cookbook_helpers (no substitution tokens) and emit it single-quoted via _bash_squote. Tests assert the hint has no expansion tokens, that no generated echo line carries backticks inside double quotes, and that bash prints the line literally. Fixes #3816 --- routes/cookbook_helpers.py | 12 +++++++ routes/cookbook_routes.py | 7 ++-- tests/test_ollama_runner_hint.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 tests/test_ollama_runner_hint.py diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index bba10e6e8..163f02cd2 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -558,6 +558,18 @@ def _bash_squote(v: str) -> str: return v.replace("'", "'\\''") +# Shown by generated runner scripts when the ollama binary is missing on the +# target host. Must stay free of backticks/$( ) and be emitted single-quoted: +# an earlier version wrapped the install one-liner in backticks inside a +# double-quoted echo, which bash executed as command substitution and ran the +# system-wide installer (including on remote SSH hosts) instead of printing +# the hint. +OLLAMA_MISSING_HINT = ( + "ERROR: Ollama not found on this server. Install it from " + "https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh" +) + + # Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve. # Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper. _SERVE_CMD_ALLOWLIST = { diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index fbd00c890..7a35de977 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) from routes.cookbook_helpers import ( _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token, _validate_local_dir, _validate_gpus, _shell_path, - _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, + _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT, _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines, _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script, load_stored_hf_token, @@ -1861,7 +1861,10 @@ def setup_cookbook_routes() -> APIRouter: runner_lines.append(' exec 3<&-; exec 3>&-') runner_lines.append('done') runner_lines.append('if ! command -v ollama &>/dev/null; then') - runner_lines.append(' echo "ERROR: Ollama not found on this server. Install it from https://ollama.com/download or `curl -fsSL https://ollama.com/install.sh | sh`."') + # Single-quoted on purpose: backticks inside a double-quoted + # echo are command substitution, and this line used to run the + # curl|sh installer on the target host instead of printing it. + runner_lines.append(f" echo '{_bash_squote(OLLAMA_MISSING_HINT)}'") runner_lines.append(' echo') runner_lines.append(' echo "=== Process exited with code 127 ==="') runner_lines.append(' exec bash -i') diff --git a/tests/test_ollama_runner_hint.py b/tests/test_ollama_runner_hint.py new file mode 100644 index 000000000..97a5d69e8 --- /dev/null +++ b/tests/test_ollama_runner_hint.py @@ -0,0 +1,57 @@ +"""The generated Ollama runner must print the install hint, not execute it. + +The runner script emitted by /api/model/serve contained: + + echo "ERROR: Ollama not found ... or `curl -fsSL .../install.sh | sh`." + +Backticks inside double quotes are bash command substitution, so on any host +without ollama the script downloaded and ran the system-wide installer +(including remote SSH serve targets) instead of printing the hint. The hint +now lives in OLLAMA_MISSING_HINT, contains no substitution tokens, and is +emitted single-quoted. +""" +import os +import shutil +import subprocess + +import pytest + +from routes.cookbook_helpers import OLLAMA_MISSING_HINT, _bash_squote + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def test_hint_has_no_shell_expansion_tokens(): + assert "`" not in OLLAMA_MISSING_HINT + assert "$(" not in OLLAMA_MISSING_HINT + + +def test_hint_still_tells_the_user_how_to_install(): + assert "https://ollama.com/download" in OLLAMA_MISSING_HINT + assert "install.sh" in OLLAMA_MISSING_HINT + + +def test_no_runner_echo_line_uses_backticks_in_double_quotes(): + # Source-level guard: generated-script echo lines must never carry + # backticks inside a double-quoted bash string again. + src = open(os.path.join(ROOT, "routes", "cookbook_routes.py"), encoding="utf-8").read() + offenders = [ + line.strip() + for line in src.splitlines() + if "append(" in line and 'echo "' in line and "`" in line.split('echo "', 1)[1] + ] + assert offenders == [] + + +def test_single_quoted_echo_prints_hint_literally(): + bash = shutil.which("bash") + if not bash: + pytest.skip("bash not available") + out = subprocess.run( + [bash, "-c", f"echo '{_bash_squote(OLLAMA_MISSING_HINT)}'"], + capture_output=True, + text=True, + timeout=30, + ) + assert out.returncode == 0 + assert out.stdout.strip() == OLLAMA_MISSING_HINT From 260f4323322fb0e94c5ef848b9b0554f73f29c82 Mon Sep 17 00:00:00 2001 From: lekt8 Date: Thu, 2 Jul 2026 17:04:22 +0800 Subject: [PATCH 03/32] fix(session): use utcnow_naive across session routes (#1116) (#5003) Replace remaining datetime.utcnow() call sites in session CRUD, incognito purge cutoff, and webhook payloads with core.database.utcnow_naive. --- routes/session_routes.py | 20 ++++++++++---------- tests/test_session_routes_utcnow.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 tests/test_session_routes_utcnow.py diff --git a/routes/session_routes.py b/routes/session_routes.py index 19b897f29..40f8d80cb 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -162,7 +162,7 @@ def _persist_session_headers(session_id: str, headers: dict | None) -> None: db_session = db.query(DbSession).filter(DbSession.id == session_id).first() if db_session: db_session.headers = headers or {} - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() db.commit() except Exception: db.rollback() @@ -223,8 +223,8 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ # purge exists only to catch ghosts the frontend missed (tab close, # crash). Only clean up rows old enough to be definitely orphaned. try: - from datetime import datetime as _dt, timedelta as _td - _cutoff = _dt.utcnow() - _td(minutes=10) + from datetime import timedelta as _td + _cutoff = utcnow_naive() - _td(minutes=10) _purge_db = SessionLocal() try: from core.database import ChatMessage as _DbMsg @@ -470,7 +470,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db_session = db.query(DbSession).filter(DbSession.id == sid).first() if db_session: db_session.folder = folder if folder else None - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() db.commit() result["folder"] = folder if folder else None finally: @@ -517,7 +517,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db_session.model = model db_session.endpoint_url = endpoint_url db_session.headers = session.headers or {} - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() db.commit() finally: db.close() @@ -646,7 +646,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db_session = db.query(DbSession).filter(DbSession.id == sid).first() if db_session: db_session.archived = True - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() db.commit() # Update in memory if it exists @@ -680,7 +680,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ if not db_session: raise HTTPException(404, f"Session {sid} not found") db_session.archived = False - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() db.commit() # Reload into session manager so it appears in the active list try: @@ -890,7 +890,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db_session = db.query(DbSession).filter(DbSession.id == session_id).first() if db_session: db_session.is_important = important - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() db.commit() # Update in memory if it exists @@ -979,7 +979,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ metadata={ "compacted": True, "summarized_count": len(older), - "timestamp": datetime.utcnow().isoformat(), + "timestamp": utcnow_naive().isoformat(), }, ) new_history = [summary_msg] + recent @@ -1256,7 +1256,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_ db_session = db_session_q.first() if db_session: db_session.folder = folder_name - db_session.updated_at = datetime.utcnow() + db_session.updated_at = utcnow_naive() updated += 1 db.commit() except Exception as e: diff --git a/tests/test_session_routes_utcnow.py b/tests/test_session_routes_utcnow.py new file mode 100644 index 000000000..33b0f18a4 --- /dev/null +++ b/tests/test_session_routes_utcnow.py @@ -0,0 +1,11 @@ +"""Regression: session routes must not call datetime.utcnow() (#1116).""" + +import inspect + +import routes.session_routes as sr + + +def test_session_routes_module_does_not_reference_utcnow(): + source = inspect.getsource(sr) + assert "datetime.utcnow()" not in source + assert "_dt.utcnow()" not in source \ No newline at end of file From b26ebbda9589cd942574261cc2645477e383798c Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:41:51 +0530 Subject: [PATCH 04/32] fix(security): match the sensitive-file deny-list case-insensitively (#5097) --- src/tool_execution.py | 26 ++++++++++++++++++-------- tests/test_tool_path_confinement.py | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/tool_execution.py b/src/tool_execution.py index a587633c9..8497b157a 100644 --- a/src/tool_execution.py +++ b/src/tool_execution.py @@ -71,25 +71,35 @@ _SENSITIVE_FILE_PATTERNS: tuple[str, ...] = ( "known_hosts", ) +# Case-folded views used for matching. On a case-insensitive filesystem +# (Windows, default macOS) ".SSH/AUTHORIZED_KEYS" and ".env" resolve to the +# same protected files as their lowercase forms, so the deny-list has to fold +# case before comparing — the sibling resolver already normcases paths for the +# same reason. casefold (not os.path.normcase) because normcase is a no-op on +# POSIX, which is exactly where the macOS read-exfil path lives. +_SENSITIVE_BASENAMES_CF: frozenset[str] = frozenset(b.casefold() for b in _SENSITIVE_BASENAMES) +_SENSITIVE_FILE_PATTERNS_CF: frozenset[str] = frozenset(p.casefold() for p in _SENSITIVE_FILE_PATTERNS) + def _is_sensitive_path(resolved: str) -> bool: """Return True if *resolved* falls under a sensitive directory or matches a sensitive filename — regardless of what root it sits under. + + Matching is case-insensitive: on Windows / default macOS a case-variant + name (``.SSH``, ``AUTHORIZED_KEYS``, ``Id_Rsa``) points at the same file as + the lowercase form, so a case-sensitive check would let it slip past the + deny-list in every file tool that relies on it. """ - parts = resolved.split(os.sep) - filenames: set[str] = {parts[-1]} if parts else set() + parts = [p.casefold() for p in resolved.split(os.sep)] + filename = parts[-1] if parts else "" # Check if any path component is a sensitive directory. for part in parts: - if part in _SENSITIVE_BASENAMES: + if part in _SENSITIVE_BASENAMES_CF: return True # Check filename against known sensitive files. - for pat in _SENSITIVE_FILE_PATTERNS: - if pat in filenames: - return True - - return False + return filename in _SENSITIVE_FILE_PATTERNS_CF def _tool_path_roots() -> list[str]: diff --git a/tests/test_tool_path_confinement.py b/tests/test_tool_path_confinement.py index 6288623c4..f9f1bd03f 100644 --- a/tests/test_tool_path_confinement.py +++ b/tests/test_tool_path_confinement.py @@ -57,6 +57,27 @@ def test_non_sensitive_path(): assert not _is_sensitive_path("/home/user/projects/file.py") +def test_sensitive_case_insensitive(): + """On case-insensitive filesystems (Windows, default macOS) a case-variant + name resolves to the same protected file, so the deny-list must match + regardless of case. Built with os.path.join so the separator is right on + both POSIX and Windows. + """ + from src.tool_execution import _is_sensitive_path + # sensitive directory, varied case + assert _is_sensitive_path(os.path.join("home", "u", ".SSH", "authorized_keys")) + assert _is_sensitive_path(os.path.join("home", "u", ".Gnupg", "pubring.kbx")) + # sensitive filename, varied case + assert _is_sensitive_path(os.path.join("ws", "AUTHORIZED_KEYS")) + assert _is_sensitive_path(os.path.join("ws", "Id_Rsa")) + assert _is_sensitive_path(os.path.join("ws", ".ENV")) + assert _is_sensitive_path(os.path.join("ws", ".Env")) + # both dir and file varied + assert _is_sensitive_path(os.path.join("home", "u", ".SSH", "AUTHORIZED_KEYS")) + # an ordinary file with none of the sensitive names is still allowed + assert not _is_sensitive_path(os.path.join("ws", "Readme.md")) + + # ── Unit tests on _resolve_tool_path ───────────────────────────────── def test_blocks_etc_shadow(): From dff91efb10e75ad03f0286ebe520cebf943edeef Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:58:33 +0530 Subject: [PATCH 05/32] fix(agent): skip deny-listed sensitive files in glob (#5094) --- src/agent_tools/filesystem_tools.py | 25 +++++++++++++++++++--- tests/test_workspace_confine.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/agent_tools/filesystem_tools.py b/src/agent_tools/filesystem_tools.py index b0f5b6a89..573d30db3 100644 --- a/src/agent_tools/filesystem_tools.py +++ b/src/agent_tools/filesystem_tools.py @@ -281,7 +281,13 @@ class LsTool: class GlobTool: async def execute(self, content: str, ctx: dict) -> dict: - from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate + from src.tool_execution import ( + _SENSITIVE_BASENAMES, + _is_sensitive_path, + _resolve_tool_path, + _resolve_search_root, + _truncate, + ) args = {} _s = (content or "").strip() if _s.startswith("{"): @@ -322,7 +328,11 @@ class GlobTool: ) == nbase except ValueError: inside = False - if inside and os.path.exists(cand): + # A literal that names a deny-listed sensitive file (.env, + # .ssh/id_rsa, …) falls through to the walk, which skips it — + # otherwise glob would surface secret paths that read_file / + # grep already refuse to touch. + if inside and os.path.exists(cand) and not _is_sensitive_path(cand): return [cand], None # Literal not at exact path — fall through to walk so # e.g. "foo.py" still matches at any depth (like rglob). @@ -334,11 +344,20 @@ class GlobTool: for dp, dns, fns in os.walk(base): # Prune skipped dirs before descending (unlike rglob which # descends first then filters — fatal on large node_modules). - dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS] + # Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob + # never enumerates the keys/tokens inside them. + dns[:] = [ + d for d in dns + if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES + ] for name in fns + dns: full = os.path.join(dp, name) rel = os.path.relpath(full, base).replace(os.sep, "/") if regex.fullmatch(rel) or regex.fullmatch(name): + # Skip deny-listed sensitive files (.env, id_rsa, + # known_hosts, …) the same way grep does. + if _is_sensitive_path(os.path.realpath(full)): + continue try: mtime = os.stat(full).st_mtime except OSError: diff --git a/tests/test_workspace_confine.py b/tests/test_workspace_confine.py index ca0819cc7..6d90789c9 100644 --- a/tests/test_workspace_confine.py +++ b/tests/test_workspace_confine.py @@ -167,6 +167,38 @@ async def test_glob_confined_e2e(ws, admin): assert r["exit_code"] == 0 and "No files" in r["output"] +@pytest.mark.asyncio +async def test_glob_skips_sensitive_files_in_workspace(ws, admin): + """glob must not enumerate deny-listed sensitive files that live inside the + workspace. read_file/write_file/edit_file refuse them and grep skips them, + so glob surfacing their paths is an enumeration oracle for prompt-injection. + """ + with open(os.path.join(ws, "keep.py"), "w") as f: + f.write("x") + with open(os.path.join(ws, ".env"), "w") as f: + f.write("AWS_SECRET=xxx") + with open(os.path.join(ws, "id_rsa"), "w") as f: # non-dotfile key at root + f.write("KEY") + os.makedirs(os.path.join(ws, ".ssh"), exist_ok=True) + with open(os.path.join(ws, ".ssh", "authorized_keys"), "w") as f: + f.write("ssh-rsa AAAA") + + # A recursive wildcard returns ordinary files but none of the sensitive + # ones. The pattern "**/*" contains no secret names, so a secret basename + # appearing in the output is a real leak (not the echoed not-found pattern). + _, r = await execute_tool_block(_block("glob", json.dumps({"pattern": "**/*"})), owner="a", workspace=ws) + assert r["exit_code"] == 0 + assert "keep.py" in r["output"] + for leak in (".env", "id_rsa", "authorized_keys"): + assert leak not in r["output"], f"glob leaked sensitive file: {leak}" + + # Directly targeting a sensitive file (literal fast-path and wildcard) must + # come back as the not-found message, never a match with the file's path. + for pat in (".env", "**/id_rsa", "**/authorized_keys"): + _, r = await execute_tool_block(_block("glob", json.dumps({"pattern": pat})), owner="a", workspace=ws) + assert r["exit_code"] == 0 and "No files" in r["output"] + + @pytest.mark.asyncio async def test_subprocess_cwd_is_workspace_e2e(ws, admin): """python tool runs with cwd = workspace (OS-agnostic probe).""" From 88191d17fb9e501769e27d910881db9dbed22826 Mon Sep 17 00:00:00 2001 From: Afonso Coutinho Date: Thu, 2 Jul 2026 10:40:19 +0100 Subject: [PATCH 06/32] fix: auto-spam move/delete targets the wrong message (seqnum vs UID) (#1874) --- routes/email_helpers.py | 9 ++++-- tests/test_imap_move_uid.py | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/test_imap_move_uid.py diff --git a/routes/email_helpers.py b/routes/email_helpers.py index 3796a6d85..b05f64681 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -1273,10 +1273,15 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str try: c = _imap_connect(account_id, owner=owner) c.select(_q(src)) - status, _ = c.copy(uid, _q(dest)) + # Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy() + # and store() operate on message SEQUENCE NUMBERS, so addressing them + # with a UID moved/deleted the wrong message (or silently no-oped when + # the UID exceeded the message count). Use the UID commands, matching + # the move/delete path in email_routes.py. + status, _ = c.uid("COPY", uid, _q(dest)) if status != "OK": return False - c.store(uid, "+FLAGS", "\\Deleted") + c.uid("STORE", uid, "+FLAGS", "\\Deleted") c.expunge() return True except Exception as e: diff --git a/tests/test_imap_move_uid.py b/tests/test_imap_move_uid.py new file mode 100644 index 000000000..4d3da1936 --- /dev/null +++ b/tests/test_imap_move_uid.py @@ -0,0 +1,56 @@ +"""_imap_move must address messages by UID, not sequence number. + +The auto-spam poller passes a real IMAP UID (from conn.uid("SEARCH", ...)) +to _imap_move, but the function used conn.copy()/conn.store(), which operate +on message SEQUENCE NUMBERS. So a UID like 90521 was interpreted as sequence +number 90521 — moving/deleting the wrong message or silently no-oping. It +must use the UID commands. +""" +import sys +import types + +import pytest + + +@pytest.fixture +def email_helpers(monkeypatch, tmp_path): + # Keep _init_scheduled_db (run at import) off the real data dir. + monkeypatch.setenv("ODYSSEUS_DATA_DIR", str(tmp_path)) + import routes.email_helpers as eh + return eh + + +class _FakeIMAP: + def __init__(self): + self.calls = [] + + def select(self, mbox): + self.calls.append(("select", mbox)); return ("OK", [b""]) + + def copy(self, *a): + self.calls.append(("copy",) + a); return ("OK", [b""]) + + def store(self, *a): + self.calls.append(("store",) + a); return ("OK", [b""]) + + def uid(self, *a): + self.calls.append(("uid",) + a); return ("OK", [b""]) + + def expunge(self): + self.calls.append(("expunge",)); return ("OK", [b""]) + + def logout(self): + pass + + +def test_move_uses_uid_commands_not_seqnum(email_helpers, monkeypatch): + fake = _FakeIMAP() + monkeypatch.setattr(email_helpers, "_imap_connect", lambda *a, **k: fake) + ok = email_helpers._imap_move(b"90521", "Spam", src="INBOX") + assert ok is True + verbs = [c[0] for c in fake.calls] + uid_ops = [c[1] for c in fake.calls if c[0] == "uid"] + assert "COPY" in uid_ops and "STORE" in uid_ops + # the sequence-number commands must NOT be used to address a UID + assert "copy" not in verbs + assert "store" not in verbs From b1f9f67d9dbf60f0e5528170665b77d014d242c5 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:58:35 +0100 Subject: [PATCH 07/32] fix(security): confine research file paths (#4986) --- routes/research/research_routes.py | 38 ++- .../test_research_routes_path_confinement.py | 273 ++++++++++++++++++ 2 files changed, 303 insertions(+), 8 deletions(-) create mode 100644 tests/test_research_routes_path_confinement.py diff --git a/routes/research/research_routes.py b/routes/research/research_routes.py index 5582f2653..b35c94b69 100644 --- a/routes/research/research_routes.py +++ b/routes/research/research_routes.py @@ -20,6 +20,26 @@ from src.constants import DEEP_RESEARCH_DIR _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") + +def _confine_research_path(session_id: str) -> Path: + """Return the resolved Path for session_id's JSON inside DEEP_RESEARCH_DIR. + + Validates the session ID format and asserts containment after symlink + expansion. Raises HTTPException(400) on format failures, traversal + attempts, absolute-path injection, and symlink escape so every caller + gets a safe, confined path with no extra validation needed. + """ + if not _SESSION_ID_RE.fullmatch(session_id): + raise HTTPException(400, "Invalid session ID") + root = Path(DEEP_RESEARCH_DIR).resolve() + candidate = (root / f"{session_id}.json").resolve() + try: + candidate.relative_to(root) + except ValueError: + raise HTTPException(400, "Invalid session ID") + return candidate + + logger = logging.getLogger(__name__) # Model-name substrings that are NOT chat/generation models — research must @@ -183,7 +203,10 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: if entry is not None: return entry.get("owner", "") == user # Task no longer in memory — check the persisted JSON. - path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" + try: + path = _confine_research_path(session_id) + except HTTPException: + return False if not path.exists(): return False try: @@ -247,7 +270,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: def _assert_owns_research(session_id: str, user: str) -> None: """404-not-403 ownership gate for a research session's on-disk JSON. Use BEFORE returning any data or mutating the file.""" - path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" + path = _confine_research_path(session_id) if not path.exists(): raise HTTPException(404, "Research not found") try: @@ -361,7 +384,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: summary, stats — used by the Library preview panel.""" user = _require_user(request) _validate_session_id(session_id) - path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" + path = _confine_research_path(session_id) if not path.exists(): raise HTTPException(404, "Research not found") try: @@ -378,7 +401,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Soft-archive / restore a research report (sets `archived` in its JSON).""" user = _require_user(request) _validate_session_id(session_id) - path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" + path = _confine_research_path(session_id) if not path.exists(): raise HTTPException(404, "Research not found") try: @@ -398,8 +421,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Delete a research result from disk.""" user = _require_user(request) _validate_session_id(session_id) - data_dir = Path(DEEP_RESEARCH_DIR) - json_path = data_dir / f"{session_id}.json" + json_path = _confine_research_path(session_id) deleted = False if json_path.exists(): # SECURITY: verify ownership before letting the caller delete it. @@ -561,7 +583,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: raise HTTPException(404, "No research found for this session") result = research_handler.get_result(session_id) if result is None: - p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" + p = _confine_research_path(session_id) if p.exists(): d = json.loads(p.read_text(encoding="utf-8")) return { @@ -601,7 +623,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: sources = research_handler.get_sources(session_id) or [] query = "" - path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" + path = _confine_research_path(session_id) if path.exists(): try: disk = json.loads(path.read_text(encoding="utf-8")) diff --git a/tests/test_research_routes_path_confinement.py b/tests/test_research_routes_path_confinement.py new file mode 100644 index 000000000..29c8a0715 --- /dev/null +++ b/tests/test_research_routes_path_confinement.py @@ -0,0 +1,273 @@ +"""Path-confinement regression tests for research routes. + +Covers the CodeQL py/path-injection alert cluster (#552-#567) in +routes/research/research_routes.py: + - _owns_in_memory disk fallback (alerts #552, #553) + - _assert_owns_research (alerts #554, #555) + - research_detail (alerts #556, #557) + - research_archive (alerts #558, #559, #560) + - research_delete (alerts #561, #562, #563) + - research_result_peek (alerts #564, #565) + - research_spinoff (alerts #566, #567) +""" + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException + +from routes.research_routes import setup_research_routes +from routes.research.research_routes import _confine_research_path + + +@pytest.fixture(autouse=True) +def _redirect_research_dir(tmp_path, monkeypatch): + monkeypatch.setattr( + "routes.research_routes.DEEP_RESEARCH_DIR", + str(tmp_path / "deep_research"), + ) + + +def _request(user: str): + return SimpleNamespace(state=SimpleNamespace(current_user=user)) + + +def _route(router, path: str, method: str): + for route in router.routes: + if getattr(route, "path", "") != path: + continue + if method in getattr(route, "methods", set()): + return route.endpoint + raise AssertionError(f"{method} {path} route not registered") + + +def _write_research(data_dir, session_id: str, **data): + data_dir.mkdir(parents=True, exist_ok=True) + path = data_dir / f"{session_id}.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + +def _research_handler(): + handler = MagicMock() + handler._active_tasks = {} + return handler + + +# --------------------------------------------------------------------------- +# Helper-level tests — _confine_research_path +# --------------------------------------------------------------------------- + +def test_confine_allows_valid_session_id(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + data_dir.mkdir() + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + path = _confine_research_path("rp-abc123de4567") + assert path == (data_dir / "rp-abc123de4567.json").resolve() + + +@pytest.mark.parametrize("bad_id", [ + "../escape", + "../../etc/passwd", + "/etc/passwd", + "safe/../../x", + "", + "rp_bad", # underscore not in allowed charset + "rp-bad.json", # dot not in allowed charset + "a" * 129, # exceeds length limit +]) +def test_confine_rejects_bad_session_ids(tmp_path, monkeypatch, bad_id): + data_dir = tmp_path / "deep_research" + data_dir.mkdir() + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + with pytest.raises(HTTPException) as exc: + _confine_research_path(bad_id) + assert exc.value.status_code == 400 + + +def test_confine_rejects_symlink_escape(tmp_path, monkeypatch): + """A symlink inside DEEP_RESEARCH_DIR that resolves outside is rejected.""" + data_dir = tmp_path / "deep_research" + outside = tmp_path / "outside" + data_dir.mkdir() + outside.mkdir() + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + target = outside / "rp-linktest1234.json" + target.write_text("{}", encoding="utf-8") + link = data_dir / "rp-linktest1234.json" + try: + link.symlink_to(target) + except (AttributeError, NotImplementedError, OSError) as e: + pytest.skip(f"symlinks unavailable: {e}") + with pytest.raises(HTTPException) as exc: + _confine_research_path("rp-linktest1234") + assert exc.value.status_code == 400 + + +# --------------------------------------------------------------------------- +# Route-level tests — valid paths work +# --------------------------------------------------------------------------- + +def test_detail_returns_data_for_owner(tmp_path): + data_dir = tmp_path / "deep_research" + _write_research(data_dir, "rp-validid12345", owner="alice", query="valid query") + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/detail/{session_id}", "GET") + out = asyncio.run(target(session_id="rp-validid12345", request=_request("alice"))) + assert out["query"] == "valid query" + + +# --------------------------------------------------------------------------- +# Route-level tests — traversal and injection rejected +# --------------------------------------------------------------------------- + +_TRAVERSAL_IDS = [ + "../escape", + "../../etc/passwd", + "/etc/passwd", + "safe/../../x", + "rp_under", + "a" * 129, +] + + +@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS) +def test_detail_rejects_traversal(bad_id): + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/detail/{session_id}", "GET") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id=bad_id, request=_request("alice"))) + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS) +def test_archive_rejects_traversal(bad_id): + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/{session_id}/archive", "POST") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id=bad_id, request=_request("alice"), archived=True)) + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS) +def test_delete_rejects_traversal(bad_id): + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/{session_id}", "DELETE") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id=bad_id, request=_request("alice"))) + assert exc.value.status_code == 400 + + +# --------------------------------------------------------------------------- +# Route-level tests — traversal does not touch files outside DEEP_RESEARCH_DIR +# --------------------------------------------------------------------------- + +def test_delete_traversal_does_not_delete_outside_file(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + data_dir.mkdir(parents=True) + outside = tmp_path / "sensitive.json" + outside.write_text('{"secret": true}', encoding="utf-8") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/{session_id}", "DELETE") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id="../sensitive", request=_request("alice"))) + assert exc.value.status_code == 400 + assert outside.exists(), "file outside DEEP_RESEARCH_DIR must not be deleted" + + +def test_archive_traversal_does_not_mutate_outside_file(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + data_dir.mkdir(parents=True) + outside = tmp_path / "sensitive.json" + outside.write_text('{"owner": "alice", "archived": false}', encoding="utf-8") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/{session_id}/archive", "POST") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id="../sensitive", request=_request("alice"), archived=True)) + assert exc.value.status_code == 400 + data = json.loads(outside.read_text(encoding="utf-8")) + assert data["archived"] is False, "file outside DEEP_RESEARCH_DIR must not be mutated" + + +def test_detail_traversal_does_not_read_outside_file(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + data_dir.mkdir(parents=True) + outside = tmp_path / "sensitive.json" + outside.write_text('{"owner": "alice", "result": "secret data"}', encoding="utf-8") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/detail/{session_id}", "GET") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id="../sensitive", request=_request("alice"))) + assert exc.value.status_code == 400 + + +# --------------------------------------------------------------------------- +# Route-level symlink escape test +# --------------------------------------------------------------------------- + +def test_detail_rejects_symlink_escape(tmp_path, monkeypatch): + """research_detail rejects a confined-format ID whose JSON is a symlink to outside.""" + data_dir = tmp_path / "deep_research" + outside_dir = tmp_path / "outside" + data_dir.mkdir(parents=True) + outside_dir.mkdir() + outside_file = outside_dir / "rp-linktest5678.json" + outside_file.write_text( + json.dumps({"owner": "alice", "result": "secret"}), encoding="utf-8" + ) + link = data_dir / "rp-linktest5678.json" + try: + link.symlink_to(outside_file) + except (AttributeError, NotImplementedError, OSError) as e: + pytest.skip(f"symlinks unavailable: {e}") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/detail/{session_id}", "GET") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id="rp-linktest5678", request=_request("alice"))) + assert exc.value.status_code == 400 + + +# --------------------------------------------------------------------------- +# Owner/session scoping cannot escape root +# --------------------------------------------------------------------------- + +def test_owner_scoped_paths_stay_within_research_root(tmp_path, monkeypatch): + """Owner-scoped session IDs never produce paths outside DEEP_RESEARCH_DIR.""" + data_dir = tmp_path / "deep_research" + data_dir.mkdir(parents=True) + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + root = data_dir.resolve() + for session_id in ("rp-abc123456789", "rp-000000000001", "abc-xyz-123"): + path = _confine_research_path(session_id) + assert path.resolve().is_relative_to(root), ( + f"{session_id!r} produced path outside research root: {path}" + ) + +@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS) +def test_result_peek_rejects_traversal(bad_id): + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/result-peek/{session_id}", "POST") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id=bad_id, request=_request("alice"))) + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS) +def test_spinoff_rejects_traversal(bad_id): + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/spinoff/{session_id}", "POST") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id=bad_id, request=_request("alice"))) + assert exc.value.status_code == 400 From 5e9b415bd928ca39ee7a702b915e9e64ed9d2bdc Mon Sep 17 00:00:00 2001 From: Ernest Hysa <59969602+ErnestHysa@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:08:06 +0100 Subject: [PATCH 08/32] fix(search): pin httpx connection to resolved IP to block DNS rebinding (#704) * fix(search): pin DNS-validated fetch connections Rebase the DNS-rebinding SSRF fix onto current dev after search content moved behind the services.search.content canonical module. Integrate the pinned httpcore NetworkBackend/BaseTransport approach with the current size-capped Client.stream fetch path, preserving Host/SNI semantics while forcing TCP connect to the already validated resolved IP. Keep src.search.content as the compatibility wrapper and preserve existing OG-image http(s) behavior; this avoids reintroducing the unrelated scope changes that previously blocked review. Add the explicit httpcore>=1.0,<2.0 requirement used by the public httpcore NetworkBackend and ConnectionPool APIs. * test(search): restore and rebase DNS rebinding regressions Keep the current security regression coverage that the stale PR branch had deleted, including auth-disabled localhost bypass and Ollama cookbook hardening tests. Carry forward the DNS-rebinding coverage for private resolve blocking, pinned TCP connect behavior, Host header preservation, redirect revalidation, and the BaseTransport/public-httpcore static guard. Update redirect tests to mock the current Client.stream-based capped fetch path rather than the older httpx.stream/get path. * test(search): adapt size-cap fetch tests to pinned client stream The DNS-rebinding repair moved _get_public_url from the module-level httpx.stream shortcut to httpx.Client(...).stream(...) so the fetch can use the pinned transport. Keep the existing size-cap test fakes by routing Client.stream through the monkeypatched httpx.stream only when a test has installed that fake; otherwise fall back to a real Client. This fixes the CI failures in tests/test_web_fetch_size_caps.py without touching unrelated upload-handler atomicity behavior, which is already flaky on clean origin/dev. --------- Co-authored-by: Alexandre Teixeira --- requirements.txt | 1 + services/search/content.py | 268 +++++++++++++++++++------ tests/test_security_regressions.py | 301 ++++++++++++++++++++++++++++- tests/test_web_fetch_size_caps.py | 51 +++++ 4 files changed, 553 insertions(+), 68 deletions(-) diff --git a/requirements.txt b/requirements.txt index 493cb5206..be5f5d450 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ uvicorn python-multipart python-dotenv httpx +httpcore>=1.0,<2.0 pydantic>=2.13.4 pydantic-settings>=2.14.1 SQLAlchemy diff --git a/services/search/content.py b/services/search/content.py index 49d050a4f..05aa23753 100644 --- a/services/search/content.py +++ b/services/search/content.py @@ -8,11 +8,13 @@ import os import re import logging import socket +import ssl from datetime import datetime, timedelta -from typing import List +from typing import Iterable, List, cast from urllib.parse import urljoin, urlparse import httpx +import httpcore from bs4 import BeautifulSoup from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT @@ -91,6 +93,148 @@ def _public_http_url(url: str) -> bool: return False +def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]: + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise httpx.RequestError(f"Blocked non-public URL: {url}") + host = (parsed.hostname or "").strip().lower() + if host in ("localhost", "metadata", "metadata.google.internal"): + raise httpx.RequestError(f"Blocked non-public hostname: {host}") + try: + ip = ipaddress.ip_address(host) + if _is_private_address(ip): + raise httpx.RequestError(f"Blocked non-public IP literal: {host}") + return [ip] + except httpx.RequestError: + raise + except ValueError: + pass + addrs = _resolve_hostname_ips(host) + if not addrs or any(_is_private_address(a) for a in addrs): + raise httpx.RequestError(f"Blocked non-public URL: {url}") + return addrs + + +class _PinnedBackend(httpcore.NetworkBackend): + """Network backend that connects to a pre-resolved IP. + + httpcore derives the TLS SNI and the ``Host`` header from the URL's + origin, not from the host argument passed to ``connect_tcp``. So + routing the TCP connect to a resolved IP while leaving the URL + untouched keeps SNI / vhost behaviour correct and closes the + DNS-rebinding TOCTOU between the SSRF check and the connect. + """ + + def __init__(self, ip: ipaddress._BaseAddress): + self._ip = str(ip) + self._real = httpcore.SyncBackend() + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options=None, + ): + return self._real.connect_tcp( + self._ip, port, timeout, local_address, socket_options + ) + + def connect_unix_socket(self, path, timeout=None, socket_options=None): + return self._real.connect_unix_socket(path, timeout, socket_options) + + def sleep(self, seconds: float) -> None: + return self._real.sleep(seconds) + + +# Map httpcore exception classes to their httpx equivalents. Built +# once at import time from the public exception classes; avoids any +# import of httpx's private transport machinery. httpcore's +# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will +# close and retry on its own) — we never expect to see it surface to +# a transport caller, so it has no httpx counterpart here. +_HTTPCORE_TO_HTTPX_EXC = { + httpcore.ConnectError: httpx.ConnectError, + httpcore.ConnectTimeout: httpx.ConnectTimeout, + httpcore.LocalProtocolError: httpx.LocalProtocolError, + httpcore.NetworkError: httpx.NetworkError, + httpcore.PoolTimeout: httpx.PoolTimeout, + httpcore.ProtocolError: httpx.ProtocolError, + httpcore.ProxyError: httpx.ProxyError, + httpcore.ReadError: httpx.ReadError, + httpcore.ReadTimeout: httpx.ReadTimeout, + httpcore.RemoteProtocolError: httpx.RemoteProtocolError, + httpcore.TimeoutException: httpx.TimeoutException, + httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol, + httpcore.WriteError: httpx.WriteError, + httpcore.WriteTimeout: httpx.WriteTimeout, +} + + +class _PinnedTransport(httpx.BaseTransport): + """Transport that pins every TCP connect to a pre-resolved IP. + + Uses only the public ``httpcore`` and ``httpx`` APIs — no + subclassing of ``httpx.HTTPTransport``, no reads of private + ``httpcore.ConnectionPool`` attributes, no imports from + ``httpx private transport internals``. The URL is passed through unchanged so SNI + / vhost work as if httpx had been given the hostname directly; + only the TCP destination is pinned, closing the DNS-rebinding + TOCTOU between the SSRF check and the connect. + """ + + def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False): + self._pool = httpcore.ConnectionPool( + ssl_context=ssl.create_default_context(), + http1=True, + http2=http2, + network_backend=_PinnedBackend(ip), + ) + + def __enter__(self): + self._pool.__enter__() + return self + + def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: + self._pool.__exit__(exc_type, exc_value, traceback) + + def handle_request(self, request: httpx.Request) -> httpx.Response: + httpcore_req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + try: + httpcore_resp = self._pool.handle_request(httpcore_req) + # Eager materialisation matches the original + # ``response.text`` usage in fetch_webpage_content. The + # sync pool's stream is a plain Iterable[bytes] despite + # the httpcore type hint unioning the async variant. + content = b"".join(cast(Iterable[bytes], httpcore_resp.stream)) + except Exception as exc: + mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc)) + if mapped is not None: + raise mapped(str(exc)) from exc + raise + + return httpx.Response( + status_code=httpcore_resp.status, + headers=httpcore_resp.headers, + content=content, + extensions=httpcore_resp.extensions, + ) + + def close(self) -> None: + self._pool.close() + class BodyTooLargeError(Exception): """The server declared a body larger than the hard fetch ceiling.""" @@ -141,78 +285,78 @@ class _CappedFetch: def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5, max_bytes: int = None) -> "_CappedFetch": - """Capped streaming GET with SSRF-guarded manual redirects. + """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects. - The body is streamed and buffering stops at ``max_bytes`` (default: the - soft cap), so an oversized resource cannot be pulled into memory or the - content cache in full. When Content-Length already declares a body over - the hard ceiling, the fetch is refused before any body bytes are read. + Each hop is resolved once, validated as public, and then the actual TCP + connection is pinned to that resolved IP. The request URL is left unchanged + so Host and TLS SNI keep the original hostname. """ cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES) current = url for _ in range(max_redirects + 1): - if not _public_http_url(current): - raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current)) + ips = _resolve_public_ips(current) + # Force identity transfer-encoding. With gzip/deflate the wire bytes - # (and Content-Length) can be a small fraction of the decoded body, so - # a tiny compressed response could pass the hard-cap preflight and then - # expand past the ceiling in a single decoded chunk before the streamed - # cap below can slice it. Identity makes Content-Length the true body - # size and keeps each streamed chunk bounded by the network read. + # and Content-Length can be a small fraction of the decoded body, so a + # tiny compressed response could pass the hard-cap preflight and then + # expand past the ceiling in one decoded chunk before the streamed cap + # below can slice it. req_headers = dict(headers or {}) req_headers["Accept-Encoding"] = "identity" - with httpx.stream("GET", current, headers=req_headers, timeout=timeout, - follow_redirects=False) as response: - if response.status_code in (301, 302, 303, 307, 308): - location = response.headers.get("location") - if not location: - return _CappedFetch(response.status_code, response.headers, b"", - False, None, response.encoding, str(response.url)) - current = urljoin(str(response.url), location) - continue - # A server can ignore the identity request and still return a - # compressed body; httpx.iter_bytes would then decode it, and a tiny - # gzip can balloon into one decoded chunk far past the cap before we - # slice. Refuse a compressed Content-Encoding so the streamed cap - # stays a real memory bound (Content-Length is the compressed wire - # length here, so the preflight and size metadata are unreliable too). - enc = (response.headers.get("content-encoding") or "").strip().lower() - if enc and enc != "identity": - raise httpx.RequestError( - f"Refusing compressed response (Content-Encoding: {enc}) after " - "requesting identity: cannot bound decoded body size", - request=httpx.Request("GET", current), - ) + with httpx.Client( + headers=req_headers, + timeout=timeout, + follow_redirects=False, + transport=_PinnedTransport(ips[0]), + ) as client: + with client.stream("GET", current) as response: + if response.status_code in (301, 302, 303, 307, 308): + location = response.headers.get("location") + if not location: + return _CappedFetch(response.status_code, response.headers, b"", + False, None, response.encoding, str(response.url)) + current = urljoin(str(response.url), location) + continue - declared = None - raw_len = response.headers.get("content-length") - if raw_len and raw_len.isdigit(): - declared = int(raw_len) - # Refuse before buffering anything when the server already tells - # us the body exceeds the absolute ceiling (Content-Length is wire - # bytes; the decompressed body can only be larger). - if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES: - raise BodyTooLargeError(current, declared) + # A server can ignore the identity request and still return a + # compressed body; httpx.iter_bytes would then decode it, and a + # tiny gzip can balloon into one decoded chunk far past the cap. + # Refuse compressed Content-Encoding so the streamed cap stays + # a real memory bound. + enc = (response.headers.get("content-encoding") or "").strip().lower() + if enc and enc != "identity": + raise httpx.RequestError( + f"Refusing compressed response (Content-Encoding: {enc}) after " + "requesting identity: cannot bound decoded body size", + request=httpx.Request("GET", current), + ) + + declared = None + raw_len = response.headers.get("content-length") + if raw_len and raw_len.isdigit(): + declared = int(raw_len) + + if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES: + raise BodyTooLargeError(current, declared) + + chunks = [] + read = 0 + truncated = False + for chunk in response.iter_bytes(): + read += len(chunk) + if read > cap: + keep = cap - (read - len(chunk)) + if keep > 0: + chunks.append(chunk[:keep]) + truncated = True + break + chunks.append(chunk) + + return _CappedFetch(response.status_code, response.headers, + b"".join(chunks), truncated, declared, + response.encoding, str(response.url)) - chunks = [] - read = 0 - truncated = False - # We requested identity above, so iter_bytes yields the raw body in - # network-read-sized chunks (no decompression expansion); the cap - # therefore bounds what we actually buffer. - for chunk in response.iter_bytes(): - read += len(chunk) - if read > cap: - keep = cap - (read - len(chunk)) - if keep > 0: - chunks.append(chunk[:keep]) - truncated = True - break - chunks.append(chunk) - return _CappedFetch(response.status_code, response.headers, - b"".join(chunks), truncated, declared, - response.encoding, str(response.url)) raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current)) # PDF extraction (optional dependency) diff --git a/tests/test_security_regressions.py b/tests/test_security_regressions.py index b8f398017..f6a05383d 100644 --- a/tests/test_security_regressions.py +++ b/tests/test_security_regressions.py @@ -894,7 +894,8 @@ def test_web_fetch_guard_fails_closed_on_empty_resolution(monkeypatch): def test_web_fetch_guard_blocks_redirect_into_private(monkeypatch): # A public URL that 302-redirects to an internal address must be blocked - # at the redirect hop, not followed. + # at the redirect hop, not followed. _get_public_url now uses + # httpx.Client(...).stream(...) so the test must mock that path. import httpx from src.search import content @@ -905,14 +906,31 @@ def test_web_fetch_guard_blocks_redirect_into_private(monkeypatch): status_code = 302 url = "http://public.example/start" headers = {"location": "http://169.254.169.254/latest/meta-data/"} + encoding = "utf-8" - from contextlib import contextmanager + class _FakeStream: + def __enter__(self): + return _Resp() - @contextmanager - def _fake_stream(method, url, **kwargs): - yield _Resp() + def __exit__(self, *args): + return False - monkeypatch.setattr(httpx, "stream", _fake_stream) + class _FakeClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def stream(self, method, url): + assert method == "GET" + assert url == "http://public.example/start" + return _FakeStream() + + monkeypatch.setattr(httpx, "Client", _FakeClient) with _pytest.raises(httpx.RequestError) as exc: content._get_public_url("http://public.example/start", headers={}, timeout=5) @@ -1224,3 +1242,274 @@ def test_visual_report_escapes_request_category(): # value must coerce rather than crash the render (html.escape needs a str). out = generate_visual_report(question="q", report_markdown="## H", category=12345) assert "category-12345" in out + + +# ── DNS rebinding (audit finding 8.1) ──────────────────────────────── +# _resolve_public_ips resolves a URL's hostname once per hop and rejects +# private / metadata targets, but httpx would then re-resolve the +# hostname at connect time. The fix: the actual TCP connect is pinned +# to the resolved IP via a custom httpcore.NetworkBackend, while the +# URL / Host header / SNI stay on the original hostname. + +import ipaddress as _ipaddr +import socket as _socket +import threading as _threading + +import httpx as _httpx + + +def test_dns_rebinding_blocked_by_resolve_gate(monkeypatch): + from src.search import content + + monkeypatch.setattr(content, "_resolve_hostname_ips", + lambda host: [_ipaddr.ip_address("10.0.0.5")]) + + with _pytest.raises(_httpx.RequestError) as exc: + content._resolve_public_ips("https://attacker.example/") + assert "non-public" in str(exc.value).lower() + + +def test_dns_rebinding_pinned_backend_connects_to_resolved_ip(monkeypatch): + """``_PinnedBackend.connect_tcp`` must ignore the URL's host and + dial the pinned IP at the original port. This is the core of the + fix: httpcore's NetworkBackend contract lets us intercept the + connect before DNS lookup happens. + """ + from src.search import content + + pinned_ip = _ipaddr.ip_address("93.184.216.34") + captured = {} + + class _StubStream: + def close(self): + pass + + class _StubBackend: + def connect_tcp(self, host, port, timeout=None, local_address=None, socket_options=None): + captured["host"] = host + captured["port"] = port + return _StubStream() + + def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise OSError("not used") + + def sleep(self, seconds): + pass + + backend = content._PinnedBackend(pinned_ip) + monkeypatch.setattr(backend, "_real", _StubBackend()) + + backend.connect_tcp("attacker.example", 443) + + assert captured["host"] == "93.184.216.34", captured + assert captured["port"] == 443, captured + + +def test_dns_rebinding_pinned_transport_dials_pinned_ip(monkeypatch): + """End-to-end: ``_PinnedTransport`` actually dials the pinned IP + when given a hostname, with the original URL's Host header + preserved. We stand up a local socket server on a free port and + make the transport connect there via the pinned backend. + """ + from src.search import content + import httpcore + + # Stand up a TCP server that accepts one connection and records + # the request bytes it received, then returns a minimal HTTP/1.1 + # response. + captured = {"request": b""} + server_sock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + server_sock.bind(("127.0.0.1", 0)) + server_sock.listen(1) + port = server_sock.getsockname()[1] + + def serve_once(): + conn, _ = server_sock.accept() + with conn: + conn.settimeout(2.0) + buf = b"" + try: + while b"\r\n\r\n" not in buf: + chunk = conn.recv(4096) + if not chunk: + break + buf += chunk + except _socket.timeout: + pass + captured["request"] = buf + conn.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 2\r\n" + b"Connection: close\r\n" + b"\r\n" + b"OK" + ) + + t = _threading.Thread(target=serve_once, daemon=True) + t.start() + + # Pin the transport to 127.0.0.1:. The caller hands it a URL + # with a fake hostname so we can verify the host header is sent + # while the TCP connect goes to the pinned IP. + pinned_ip = _ipaddr.ip_address("127.0.0.1") + transport = content._PinnedTransport(pinned_ip) + + req = _httpx.Request( + "GET", + f"http://attacker.test:{port}/path?q=1", + headers={"host": "attacker.test"}, + ) + try: + with _httpx.Client(transport=transport, timeout=5) as client: + response = client.send(req) + assert response.status_code == 200, response.text + finally: + server_sock.close() + + t.join(timeout=2) + + request_bytes = captured["request"] + assert request_bytes, "server never received a request" + # Host header is the original hostname, not the IP. (httpx + # lowercases header names; compare case-insensitively.) + headers_blob = request_bytes.lower() + assert b"host: attacker.test" in headers_blob, request_bytes + # The path was preserved. + assert b"/path?q=1" in request_bytes, request_bytes + + +def test_dns_rebinding_pinned_transport_preserves_url_netloc(monkeypatch): + """The URL the transport hands to the underlying httpcore layer + must still be the original ``https://example.com/...`` — never + rewritten to the pinned IP. SNI / vhost depend on this. + """ + from src.search import content + + seen_url = {} + + class _RecordingPool: + def handle_request(self, req): + seen_url["host"] = req.url.host.decode() if isinstance(req.url.host, bytes) else req.url.host + seen_url["scheme"] = req.url.scheme.decode() if isinstance(req.url.scheme, bytes) else req.url.scheme + seen_url["target"] = req.url.target.decode() if isinstance(req.url.target, bytes) else req.url.target + raise _httpx.ConnectError("intercepted") + + def close(self): + pass + + pinned_ip = _ipaddr.ip_address("93.184.216.34") + transport = content._PinnedTransport(pinned_ip) + transport._pool = _RecordingPool() + + req = _httpx.Request("GET", "https://example.com/some/path?q=1") + with _pytest.raises(_httpx.ConnectError): + transport.handle_request(req) + + assert seen_url["host"] == "example.com", seen_url + assert seen_url["scheme"] == "https", seen_url + assert seen_url["target"] == "/some/path?q=1", seen_url + + +def test_dns_rebinding_redirect_re_resolves_per_hop(monkeypatch): + """Every redirect hop must call ``_resolve_public_ips`` again. + A redirect to a private-IP target must be blocked even when the + first hop was public. + """ + from src.search import content + + seen = [] + + def fake_resolve(url): + seen.append(url) + if "private" in url: + raise _httpx.RequestError(f"Blocked non-public URL: {url}") + return [_ipaddr.ip_address("93.184.216.34")] + + monkeypatch.setattr(content, "_resolve_public_ips", fake_resolve) + + class _Resp: + status_code = 302 + headers = {"location": "http://private.example/secret"} + encoding = "utf-8" + + def __init__(self, url): + self.url = url + + class _FakeStream: + def __init__(self, response): + self.response = response + + def __enter__(self): + return self.response + + def __exit__(self, *args): + return False + + class _FakeClient: + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def stream(self, method, url): + assert method == "GET" + return _FakeStream(_Resp(url)) + + monkeypatch.setattr(_httpx, "Client", _FakeClient) + + with _pytest.raises(_httpx.RequestError) as exc: + content._get_public_url("http://public.example/start", headers={}, timeout=5) + assert "non-public" in str(exc.value).lower() + # Both hops were validated. + assert seen == ["http://public.example/start", "http://private.example/secret"], seen + + +def test_dns_rebinding_transport_uses_public_apis(monkeypatch): + """Static guard: ``_PinnedTransport`` must use only the public + ``httpx.BaseTransport`` / ``httpcore`` APIs. No subclassing of + ``httpx.HTTPTransport`` (whose ``_pool`` slot we'd have to + overwrite), no reads of private ``httpcore.ConnectionPool`` + attributes, and no imports from ``httpx._transports``. + """ + from src.search import content + + import inspect + + # 1) Subclass check: must be BaseTransport, not HTTPTransport. + mro_names = [c.__name__ for c in content._PinnedTransport.__mro__] + assert "BaseTransport" in mro_names, mro_names + assert "HTTPTransport" not in mro_names, ( + "_PinnedTransport subclasses httpx.HTTPTransport. Subclass " + "httpx.BaseTransport instead and build the pool from scratch " + "with the public httpcore.ConnectionPool API." + ) + + # 2) No reads of private httpcore.ConnectionPool attrs. + src = inspect.getsource(content._PinnedTransport) + forbidden = ( + "_ssl_context", + "_max_connections", + "_max_keepalive_connections", + "_keepalive_expiry", + "_http1", + "_http2", + "_network_backend", + ) + leaked = [name for name in forbidden if name in src] + assert not leaked, ( + f"_PinnedTransport reads private httpcore.ConnectionPool attrs: {leaked}. " + "Build the pool from the public httpcore.ConnectionPool API instead." + ) + + # 3) No imports from httpx's private transport module. + module_src = inspect.getsource(content) + forbidden_imports = ("from httpx._transports", "import httpx._transports") + leaked_imports = [s for s in forbidden_imports if s in module_src] + assert not leaked_imports, ( + f"content.py imports from httpx's private transport module: {leaked_imports}. " + "Use only the public httpx and httpcore APIs." + ) diff --git a/tests/test_web_fetch_size_caps.py b/tests/test_web_fetch_size_caps.py index 19320c6c2..a3cfa64ed 100644 --- a/tests/test_web_fetch_size_caps.py +++ b/tests/test_web_fetch_size_caps.py @@ -13,6 +13,57 @@ import pytest from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES from services.search import content as content_mod +import pytest as _pytest_for_client_stream_compat + + +@_pytest_for_client_stream_compat.fixture(autouse=True) +def _client_stream_compat_for_pinned_fetch(monkeypatch): + """Adapt old size-cap tests to the current pinned Client.stream path. + + These tests monkeypatch httpx.stream(...) to return fake responses. The + production fetcher now uses httpx.Client(...).stream(...) so it can pass a + pinned transport. When a test has replaced httpx.stream, route Client.stream + through that fake. When it has not, fall back to a real Client so unrelated + behavior in this file is not changed. + """ + import httpx + + real_client_cls = httpx.Client + original_stream = httpx.stream + + class _ClientProxy: + def __init__(self, *args, **kwargs): + self._args = args + self._kwargs = kwargs + self._real_cm = None + self._real_client = None + + def __enter__(self): + if httpx.stream is original_stream: + self._real_cm = real_client_cls(*self._args, **self._kwargs) + self._real_client = self._real_cm.__enter__() + return self._real_client + return self + + def __exit__(self, *args): + if self._real_cm is not None: + return self._real_cm.__exit__(*args) + return False + + def stream(self, method, url): + if self._real_client is not None: + return self._real_client.stream(method, url) + + kwargs = { + "headers": self._kwargs.get("headers"), + "timeout": self._kwargs.get("timeout"), + "follow_redirects": self._kwargs.get("follow_redirects"), + } + return httpx.stream(method, url, **kwargs) + + monkeypatch.setattr(httpx, "Client", _ClientProxy) + + class _FakeStream: """Stands in for the httpx.stream(...) context manager.""" From 0dc98ec9b9af98199260cfca3d0e96b2b58c7258 Mon Sep 17 00:00:00 2001 From: holden093 Date: Thu, 2 Jul 2026 17:05:55 +0200 Subject: [PATCH 09/32] fix(ui): prevent race condition in default chat model dropdown init (#5024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting epSel.value triggered an async change event whose handler called refreshModels('') — wiping the correct model selection that refreshModels(settings.default_model) had just applied moments earlier. The dropdown silently fell back to the alphabetically-first model (deepseek-v4-flash instead of qwen-3.6-35B-A3B). Moved the change listener registration to after the settings block so the async change event fires before any listener exists. The utility and teacher sections already followed this pattern. --- static/js/settings.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/static/js/settings.js b/static/js/settings.js index 4bff7ad8c..656ef65de 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -542,6 +542,9 @@ async function initDefaultChat() { renderFallbacks(); } catch (e) { console.warn('Failed to load default chat settings', e); } + epSel.addEventListener('change', function() { refreshModels(''); saveDefault(); }); + modelSel.addEventListener('change', saveDefault); + async function saveDefault() { try { var clean = _fallbacks.filter(function(f) { return f.endpoint_id && f.model; }); @@ -558,8 +561,6 @@ async function initDefaultChat() { } catch (e) { msg.textContent = 'Failed to save'; msg.style.color = 'var(--red)'; } } - epSel.addEventListener('change', function() { refreshModels(''); saveDefault(); }); - modelSel.addEventListener('change', saveDefault); if (addFbBtn) addFbBtn.addEventListener('click', function() { var first = enabledEndpoints()[0]; _fallbacks.push({ endpoint_id: first ? first.id : '', model: '' }); From 8c943226f8159811fb2a0592a5475dd14c67c1e1 Mon Sep 17 00:00:00 2001 From: Moniz <44476844+BrunooMoniz@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:28:23 -0300 Subject: [PATCH 10/32] fix(mobile): stack the model-comparison grid into one column on phones (#4979) The comparison grid hard-codes 2-4 equal columns with no phone breakpoint, so at 390px two models get ~178px columns and four get ~88px columns. Each column is a full scrolling chat, so content is unreadably over-wrapped and clipped. On phones (<=768px), stack the panes into a single scrollable column. Desktop is unaffected. Co-authored-by: Claude Opus 4.8 (1M context) --- static/style.css | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/static/style.css b/static/style.css index 3d48db469..d5de9a465 100644 --- a/static/style.css +++ b/static/style.css @@ -39695,3 +39695,13 @@ body.theme-frosted .modal { .log-line-default { color: var(--fg, #9cdef2); } + +/* The model-comparison grid hard-codes 2-4 equal columns with no phone + breakpoint that stacks them, so at 390px two models render ~178px columns and + four render ~88px columns. Each column is a full scrolling chat (code blocks, + tool output, vote footer), so the text is unreadably over-wrapped and clipped. + On phones, stack the panes into a single scrollable column instead. */ +@media (max-width: 768px) { + .compare-grid[data-cols] { grid-template-columns: 1fr !important; overflow-y: auto; } + .compare-pane { min-height: 60dvh; } +} From 7f43678a244730d689ad924ab146e65f9dcece53 Mon Sep 17 00:00:00 2001 From: Tanmay Garg <102200932+Tanmay9223@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:37:44 +0530 Subject: [PATCH 11/32] fix(tools): handle non-dict JSON values in _parse_tool_args (closes #5043) (#5064) When an LLM generates a valid JSON string that parses to a native non-dict type (like a list, int, or string), _parse_tool_args previously returned that object. Callers expecting a dictionary would then crash with AttributeError or KeyError when attempting to look up action keys. - Update _parse_tool_args in src/tool_utils.py to explicitly type-check the parsed JSON object and return {} for non-dict objects. - Add test coverage in tests/test_admin_tools_registry.py for lists, ints, and strings. --- src/tool_utils.py | 2 ++ tests/test_admin_tools_registry.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/tool_utils.py b/src/tool_utils.py index bb60a1095..8255bc0a9 100644 --- a/src/tool_utils.py +++ b/src/tool_utils.py @@ -54,6 +54,8 @@ def _parse_tool_args(content): if isinstance(content, str): try: args = json.loads(content) if content.strip() else {} + if not isinstance(args, dict): + args = {} except (json.JSONDecodeError, TypeError) as e: raise ValueError(str(e)) elif isinstance(content, dict): diff --git a/tests/test_admin_tools_registry.py b/tests/test_admin_tools_registry.py index 4bde9ea9e..0470a0e36 100644 --- a/tests/test_admin_tools_registry.py +++ b/tests/test_admin_tools_registry.py @@ -72,3 +72,8 @@ def test_parse_tool_args_lives_in_tool_utils_single_source(): assert _parse_tool_args('{"action":"add"}') == {"action": "add"} # body-envelope unwrap still works assert _parse_tool_args('{"body":{"action":"x"}}') == {"action": "x"} + + # non-dict JSON values should return {} + assert _parse_tool_args('[1, 2]') == {} + assert _parse_tool_args('42') == {} + assert _parse_tool_args('"hello"') == {} From ff7164b9ec0d056e0cbc2facfd699fe1db4ae562 Mon Sep 17 00:00:00 2001 From: Abdul Fatah Jamro Date: Fri, 3 Jul 2026 15:07:16 +0100 Subject: [PATCH 12/32] fix: resolve RAG manager search signature TypeError (#4994) * fix: resolve RAG manager search signature TypeError and adjust similarity threshold * fix: revert similarity threshold change to keep PR focused * test(rag): remove trailing whitespace --------- Co-authored-by: Alexandre Teixeira --- src/rag_manager.py | 4 ++-- tests/test_rag_search_signature.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/test_rag_search_signature.py diff --git a/src/rag_manager.py b/src/rag_manager.py index a41608ecf..887030ee0 100644 --- a/src/rag_manager.py +++ b/src/rag_manager.py @@ -32,9 +32,9 @@ class RAGManager: logger.info("RAGManager initialized as wrapper for VectorRAG") # Delegate all methods to VectorRAG - def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]: + def search(self, query: str, k: int = 5, owner: Optional[str] = None) -> List[Dict[str, Any]]: """Search for documents - delegates to VectorRAG.""" - return self.vector_rag.search(query, k) + return self.vector_rag.search(query, k, owner=owner) def index_personal_documents( self, diff --git a/tests/test_rag_search_signature.py b/tests/test_rag_search_signature.py new file mode 100644 index 000000000..eb6dbcd67 --- /dev/null +++ b/tests/test_rag_search_signature.py @@ -0,0 +1,22 @@ +import unittest +from unittest.mock import MagicMock, patch +from src.rag_manager import RAGManager + +class TestRAGManagerSearchSignature(unittest.TestCase): + @patch('src.rag_manager.VectorRAG') + def test_search_signature_accepts_owner(self, mock_vector_rag_class): + # Create a mock instance for VectorRAG + mock_vector_rag = MagicMock() + mock_vector_rag_class.return_value = mock_vector_rag + + # Initialize RAGManager + manager = RAGManager() + + # Test call with owner parameter + manager.search("test query", k=3, owner="user1") + + # Verify that search was called on the underlying vector_rag with the correct parameters + mock_vector_rag.search.assert_called_once_with("test query", 3, owner="user1") + +if __name__ == '__main__': + unittest.main() From b7df800e947f95615ce2833672201920c459c56a Mon Sep 17 00:00:00 2001 From: Christer Hantilson Date: Fri, 3 Jul 2026 15:46:14 +0200 Subject: [PATCH 13/32] fix(agent): fall back to keyword tool selection when retrieval times out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retrieval-timeout branch hard-coded ALWAYS_AVAILABLE, silently skipping the deterministic keyword hints whenever the embedding backend was slow (e.g. a remote endpoint cold-loading its model). Queries that named email or calendar outright lost those tools and the model concluded the integrations did not exist. Let the timeout fall through to the existing keyword fallback instead — same baseline, plus the hints. Co-Authored-By: Claude Fable 5 --- src/agent_loop.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/agent_loop.py b/src/agent_loop.py index e3fa1f0b6..a0be4eb08 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -2532,11 +2532,17 @@ async def stream_agent_loop( ) logger.info(f"[tool-rag] Retrieved tools for query: {sorted(_relevant_tools - ALWAYS_AVAILABLE)}") except asyncio.TimeoutError: + # Leave _relevant_tools unset so the keyword fallback + # below still runs. Hard-coding ALWAYS_AVAILABLE here + # skipped the deterministic keyword hints whenever the + # embedding backend was slow (e.g. a remote endpoint + # cold-loading its model), silently stripping email/ + # calendar tools from queries that named them outright. logger.warning( - "[tool-rag] Retrieval exceeded %.1fs; falling back to always-available tools", + "[tool-rag] Retrieval exceeded %.1fs; falling back to keyword tool selection", _TOOL_SELECTION_TIMEOUT_SECONDS, ) - _relevant_tools = set(ALWAYS_AVAILABLE) + _relevant_tools = None except Exception as e: logger.warning(f"[tool-rag] Retrieval failed, using keyword fallback: {e}") _relevant_tools = None From 0b3338c69d58fc95ae9f1e29c6da044c8c6d56e6 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:50:49 +0100 Subject: [PATCH 14/32] test: split service health tests (#4972) * test: split service health tests * test(service-health): preserve focus selector --- tests/run_focus.py | 9 + tests/test_run_focus.py | 42 +++ tests/test_service_health.py | 472 ------------------------- tests/test_service_health_chromadb.py | 37 ++ tests/test_service_health_collect.py | 139 ++++++++ tests/test_service_health_email.py | 80 +++++ tests/test_service_health_ntfy.py | 62 ++++ tests/test_service_health_providers.py | 100 ++++++ tests/test_service_health_search.py | 79 +++++ 9 files changed, 548 insertions(+), 472 deletions(-) delete mode 100644 tests/test_service_health.py create mode 100644 tests/test_service_health_chromadb.py create mode 100644 tests/test_service_health_collect.py create mode 100644 tests/test_service_health_email.py create mode 100644 tests/test_service_health_ntfy.py create mode 100644 tests/test_service_health_providers.py create mode 100644 tests/test_service_health_search.py diff --git a/tests/run_focus.py b/tests/run_focus.py index 0e0066660..b0a0d6100 100644 --- a/tests/run_focus.py +++ b/tests/run_focus.py @@ -50,6 +50,14 @@ AREAS: tuple[str, ...] = ( # Backward-compatible aggregate selectors for focused runs whose original # monolithic files were split into more specific taxonomy sub-areas. SUB_AREA_ALIASES: dict[str, tuple[str, ...]] = { + "service_health": ( + "service_health_chromadb", + "service_health_search", + "service_health_ntfy", + "service_health_email", + "service_health_providers", + "service_health_collect", + ), "embedding": ("embedding", "embedding_memory"), } @@ -214,6 +222,7 @@ def build_parser( """Build the argument parser for the focused runner.""" if valid_sub_areas is None: valid_sub_areas = discover_sub_areas() + valid_sub_areas = frozenset(valid_sub_areas) | frozenset(SUB_AREA_ALIASES) parser = argparse.ArgumentParser( prog="run_focus.py", description=( diff --git a/tests/test_run_focus.py b/tests/test_run_focus.py index d78cd01eb..c1c2797f2 100644 --- a/tests/test_run_focus.py +++ b/tests/test_run_focus.py @@ -448,3 +448,45 @@ def test_fast_lane_collects_only_unmarked_auth_concurrency_test(): assert _FAST_AUTH_CONCURRENCY_TEST in collected for slow_test in _SLOW_AUTH_CONCURRENCY_TESTS: assert slow_test not in collected, f"slow test was not deselected: {slow_test}" + +def test_service_health_sub_area_command_includes_split_files(): + assert _cmd(sub_area="service_health") == [ + PY, + "-m", + "pytest", + "-m", + ( + "(sub_service_health_chromadb or " + "sub_service_health_search or " + "sub_service_health_ntfy or " + "sub_service_health_email or " + "sub_service_health_providers or " + "sub_service_health_collect)" + ), + ] + + +def test_service_health_alias_is_accepted_by_run(): + seen = [] + + def executor(cmd): + seen.append(cmd) + return 0 + + result = run(["--sub-area", "service_health"], executor=executor) + + assert result == 0 + assert len(seen) == 1 + assert seen[0][1:] == [ + "-m", + "pytest", + "-m", + ( + "(sub_service_health_chromadb or " + "sub_service_health_search or " + "sub_service_health_ntfy or " + "sub_service_health_email or " + "sub_service_health_providers or " + "sub_service_health_collect)" + ), + ] diff --git a/tests/test_service_health.py b/tests/test_service_health.py deleted file mode 100644 index 56283cef8..000000000 --- a/tests/test_service_health.py +++ /dev/null @@ -1,472 +0,0 @@ -"""Tests for src.service_health — the consolidated degraded-state report. - -Imports the real module (conftest.py stubs the heavy deps). Network is never -touched: HTTP probes take an injected `http_get`, and the email/provider probes -take an injected `connect` / `probe`. Asserts the ok/degraded/down/disabled -mapping per subsystem, the overall rollup, and that no secrets leak into meta. -""" -import types - -import pytest - -from src import service_health as sh - - -def _resp(status_code): - return types.SimpleNamespace(status_code=status_code) - - -def _raise(*_a, **_k): - raise RuntimeError("connection refused") - - -# ── chromadb_health ── - -class _Store: - def __init__(self, healthy): - self.healthy = healthy - - -def test_chromadb_both_healthy_ok(): - s = sh.chromadb_health(_Store(True), _Store(True)) - assert s["status"] == sh.OK - assert s["meta"] == {"rag": True, "memory": True} - - -def test_chromadb_one_down_degraded(): - s = sh.chromadb_health(_Store(True), _Store(False)) - assert s["status"] == sh.DEGRADED - - -def test_chromadb_both_unhealthy_down(): - s = sh.chromadb_health(_Store(False), _Store(False)) - assert s["status"] == sh.DOWN - - -def test_chromadb_both_absent_disabled(): - s = sh.chromadb_health(None, None) - assert s["status"] == sh.DISABLED - - -def test_chromadb_one_absent_one_healthy_ok(): - # An absent store is not a failure; the present one being healthy is ok. - s = sh.chromadb_health(_Store(True), None) - assert s["status"] == sh.OK - assert s["meta"]["memory"] is None - - -# ── searxng_health ── - -def test_searxng_disabled_when_other_provider(): - s = sh.searxng_health({"search_provider": "brave"}) - assert s["status"] == sh.DISABLED - - -def test_searxng_ok_on_healthz(): - s = sh.searxng_health( - {"search_provider": "searxng", "search_url": "http://sx:8080"}, - http_get=lambda url, timeout: _resp(200), - ) - assert s["status"] == sh.OK - assert s["meta"]["probed"] == "/healthz" - - -def test_searxng_ok_on_root_fallback(): - def getter(url, timeout): - return _resp(404) if url.endswith("/healthz") else _resp(200) - - s = sh.searxng_health( - {"search_provider": "searxng", "search_url": "http://sx:8080"}, - http_get=getter, - ) - assert s["status"] == sh.OK - assert s["meta"]["probed"] == "/" - - -def test_searxng_down_on_exception(): - s = sh.searxng_health( - {"search_provider": "searxng", "search_url": "http://sx:8080"}, - http_get=_raise, - ) - assert s["status"] == sh.DOWN - - -def test_searxng_down_on_5xx(): - s = sh.searxng_health( - {"search_provider": "searxng", "search_url": "http://sx:8080"}, - http_get=lambda url, timeout: _resp(502), - ) - assert s["status"] == sh.DOWN - - -# ── ntfy_health ── - -def _ntfy_intg(): - return [{"preset": "ntfy", "enabled": True, "base_url": "http://ntfy:80"}] - - -def test_ntfy_disabled_without_integration(): - s = sh.ntfy_health([], {"reminder_channel": "ntfy"}) - assert s["status"] == sh.DISABLED - - -def test_ntfy_ok(): - s = sh.ntfy_health(_ntfy_intg(), {"reminder_channel": "ntfy"}, - http_get=lambda url, timeout: _resp(200)) - assert s["status"] == sh.OK - assert s["meta"]["base"] == "http://ntfy:80" - - -def test_ntfy_probes_v1_health_not_a_topic(): - seen = {} - - def getter(url, timeout): - seen["url"] = url - return _resp(200) - - sh.ntfy_health(_ntfy_intg(), {"reminder_channel": "ntfy"}, http_get=getter) - # Non-intrusive: hits /v1/health, never publishes to a topic. - assert seen["url"].endswith("/v1/health") - - -def test_ntfy_down_on_exception(): - s = sh.ntfy_health(_ntfy_intg(), {"reminder_channel": "ntfy"}, - http_get=_raise) - assert s["status"] == sh.DOWN - - -# ── email_health ── - -def _acct(name, host="imap.example.com"): - return {"account_id": name, "account_name": name, "imap_host": host, - "imap_password": "hunter2"} - - -class _Conn: - def logout(self): - pass - - -def test_email_disabled_without_accounts(): - assert sh.email_health([])["status"] == sh.DISABLED - - -def test_email_ok_all_connect(): - s = sh.email_health([_acct("a"), _acct("b")], connect=lambda _id: _Conn()) - assert s["status"] == sh.OK - - -def test_email_degraded_some_fail(): - def connect(account_id): - if account_id == "bad": - raise RuntimeError("auth failed") - return _Conn() - - s = sh.email_health([_acct("good"), _acct("bad")], connect=connect) - assert s["status"] == sh.DEGRADED - - -def test_email_down_all_fail(): - s = sh.email_health([_acct("a")], connect=_raise) - assert s["status"] == sh.DOWN - - -def test_email_account_without_host_marked_failed(): - s = sh.email_health([_acct("a", host="")], connect=lambda _id: _Conn()) - assert s["status"] == sh.DOWN - - -def test_email_meta_never_leaks_password(): - s = sh.email_health([_acct("a")], connect=lambda _id: _Conn()) - assert "hunter2" not in repr(s) - - -# ── providers_health ── - -def _ep(name): - return {"name": name, "base_url": f"http://{name}:8000/v1", "api_key": "sk-secret"} - - -def test_providers_disabled_without_endpoints(): - assert sh.providers_health([])["status"] == sh.DISABLED - - -def test_providers_ok_all_reachable(): - s = sh.providers_health([_ep("a")], - probe=lambda base, key, timeout: ["m1", "m2"]) - assert s["status"] == sh.OK - assert s["meta"]["endpoints"][0]["model_count"] == 2 - - -def test_providers_degraded_some_empty(): - def probe(base, key, timeout): - return ["m1"] if "good" in base else [] - - s = sh.providers_health([_ep("good"), _ep("bad")], probe=probe) - assert s["status"] == sh.DEGRADED - - -def test_providers_down_all_fail(): - s = sh.providers_health([_ep("a")], probe=_raise) - assert s["status"] == sh.DOWN - - -def test_providers_meta_never_leaks_api_key(): - s = sh.providers_health([_ep("a")], - probe=lambda base, key, timeout: ["m1"]) - assert "sk-secret" not in repr(s) - - -# ── rollup ── - -def test_rollup_picks_worst_non_disabled(): - services = [ - {"status": sh.OK}, {"status": sh.DISABLED}, - {"status": sh.DEGRADED}, {"status": sh.OK}, - ] - assert sh._rollup(services) == sh.DEGRADED - - -def test_rollup_down_beats_degraded(): - assert sh._rollup([{"status": sh.DEGRADED}, {"status": sh.DOWN}]) == sh.DOWN - - -def test_rollup_all_disabled_is_ok(): - assert sh._rollup([{"status": sh.DISABLED}, {"status": sh.DISABLED}]) == sh.OK - - -# ── collect_service_health (async aggregate) ── - -def test_collect_service_health_shape(monkeypatch): - import asyncio - - # Avoid touching real data sources / network. - monkeypatch.setattr(sh, "_gather_inputs", lambda: { - "settings": {"search_provider": "disabled"}, - "integrations": [], - "accounts": [], - "endpoints": [], - }) - out = asyncio.run(sh.collect_service_health(_Store(True), _Store(True))) - assert set(out) == {"overall", "services", "timestamp"} - names = {s["name"] for s in out["services"]} - assert names == {"chromadb", "searxng", "ntfy", "email", "providers"} - # Chroma healthy, everything else disabled → overall ok. - assert out["overall"] == sh.OK - - -# ── _safe_url: strip userinfo / query / fragment ── - -@pytest.mark.parametrize("raw,expected", [ - ("http://user:pass@host:8080/path?api_key=secret#frag", "http://host:8080/path"), - ("https://admin:hunter2@searx.example.com/", "https://searx.example.com"), - ("http://ntfy.local:80?token=abc", "http://ntfy.local:80"), - ("host:8080", "host:8080"), - ("", ""), - (None, ""), -]) -def test_safe_url_strips_secrets(raw, expected): - out = sh._safe_url(raw) - assert out == expected - for bad in ("pass", "secret", "hunter2", "abc", "token", "@"): - if raw and bad in raw and bad not in expected: - assert bad not in out - - -# ── _classify_error: controlled categories, never raw text ── - -def test_classify_error_categories(): - import socket - assert sh._classify_error(TimeoutError()) == "timeout" - assert sh._classify_error(socket.timeout()) == "timeout" - assert sh._classify_error(socket.gaierror()) == "dns_error" - assert sh._classify_error(ConnectionRefusedError()) == "connection_refused" - assert sh._classify_error(OSError("boom")) == "network_error" - assert sh._classify_error(ValueError("x")) == "error" - - -# ── Sanitization in subsystem output (blocker #2) ── - -def test_searxng_meta_redacts_instance_url(): - s = sh.searxng_health( - {"search_provider": "searxng", - "search_url": "http://user:s3cr3t@searx.local:8080/?token=zzz"}, - http_get=lambda url, timeout: _resp(200), - ) - blob = repr(s) - assert "s3cr3t" not in blob and "zzz" not in blob and "user:" not in blob - assert s["meta"]["instance"] == "http://searx.local:8080" - - -def test_searxng_down_uses_error_category_not_raw_exception(): - def boom(url, timeout): - raise RuntimeError("failed connecting to http://user:pw@searx.local secret-token") - s = sh.searxng_health( - {"search_provider": "searxng", "search_url": "http://searx.local"}, - http_get=boom, - ) - assert s["status"] == sh.DOWN - assert s["meta"]["error"] == "error" # controlled category token - assert "secret-token" not in repr(s) and "pw@" not in repr(s) - - -def test_ntfy_meta_redacts_userinfo_in_base(): - intg = [{"preset": "ntfy", "enabled": True, - "base_url": "https://user:topsecret@ntfy.example.com"}] - seen = {} - - def getter(url, timeout): - seen["url"] = url # the probe itself may keep credentials - return _resp(200) - - s = sh.ntfy_health(intg, {"reminder_channel": "ntfy"}, http_get=getter) - assert s["meta"]["base"] == "https://ntfy.example.com" - assert "topsecret" not in repr(s) - - -def test_providers_name_fallback_is_sanitized(): - # No display name → falls back to the base_url, which must be sanitized. - ep = {"base_url": "http://user:k3y@prov.local:9000/v1?api_key=zzz", "api_key": "sk-x"} - s = sh.providers_health([ep], probe=lambda b, k, t: ["m1"]) - entry = s["meta"]["endpoints"][0] - assert entry["name"] == "http://prov.local:9000/v1" - assert "k3y" not in repr(s) and "zzz" not in repr(s) and "sk-x" not in repr(s) - - -def test_providers_probe_exception_maps_to_category(): - def boom(base, key, timeout): - raise RuntimeError(f"500 from {base} with key {key}") # would leak base+key - s = sh.providers_health([_ep("a")], probe=boom) - assert s["status"] == sh.DOWN - assert s["meta"]["endpoints"][0]["error"] == "error" - assert "sk-secret" not in repr(s) and "http://a" not in repr(s) - - -def test_email_connect_exception_maps_to_category(): - def boom(account_id): - raise RuntimeError("login failed for user bob with password hunter2") - s = sh.email_health([_acct("a")], connect=boom) - assert s["status"] == sh.DOWN - assert s["meta"]["accounts"][0]["error"] == "error" - assert "hunter2" not in repr(s) - - -# ── Bounded wall-clock (blocker #1) ── - -def test_providers_bounded_marks_slow_as_timeout(monkeypatch): - import time - monkeypatch.setattr(sh, "_FANOUT_BUDGET", 1) - - def probe(base, key, timeout): - if "slow" in base: - time.sleep(10) # would blow the budget if unbounded - return ["m1"] - - eps = [{"name": "fast", "base_url": "http://fast", "api_key": "k"}, - {"name": "slow", "base_url": "http://slow", "api_key": "k"}] - t0 = time.monotonic() - out = sh.providers_health(eps, probe=probe) - elapsed = time.monotonic() - t0 - assert elapsed < 4, f"providers_health not bounded: took {elapsed:.1f}s" - by = {e["name"]: e for e in out["meta"]["endpoints"]} - assert by["fast"]["ok"] is True - assert by["slow"]["ok"] is False and by["slow"]["error"] == "timeout" - assert out["status"] == sh.DEGRADED - - -def test_providers_bounded_with_many_slow_endpoints(monkeypatch): - import time - monkeypatch.setattr(sh, "_FANOUT_BUDGET", 1) - - def probe(base, key, timeout): - time.sleep(10) - return ["m1"] - - eps = [{"name": f"ep{i}", "base_url": f"http://ep{i}", "api_key": "k"} - for i in range(25)] - t0 = time.monotonic() - out = sh.providers_health(eps, probe=probe) - elapsed = time.monotonic() - t0 - # 25 endpoints * sleep would be huge if sequential; bounded keeps it ~budget. - assert elapsed < 4, f"not bounded with many endpoints: {elapsed:.1f}s" - assert out["status"] == sh.DOWN - assert all(e["error"] == "timeout" for e in out["meta"]["endpoints"]) - - -def test_email_bounded_marks_slow_as_timeout(monkeypatch): - import time - monkeypatch.setattr(sh, "_FANOUT_BUDGET", 1) - - def connect(account_id): - if account_id == "slow": - time.sleep(10) - return _Conn() - - accts = [_acct("fast"), _acct("slow")] - accts[1]["account_id"] = "slow" - t0 = time.monotonic() - out = sh.email_health(accts, connect=connect) - elapsed = time.monotonic() - t0 - assert elapsed < 4, f"email_health not bounded: took {elapsed:.1f}s" - by = {a["name"]: a for a in out["meta"]["accounts"]} - assert by["slow"]["error"] == "timeout" - - -def test_collect_runs_subsystems_concurrently(monkeypatch): - # The aggregate is bounded by running the (internally-bounded) subsystems - # concurrently, so total wall-clock ≈ max(subsystem), not the sum. Each of - # the four network subsystems here sleeps ~0.6s; sequential would be ~2.4s. - import asyncio - import time - monkeypatch.setattr(sh, "_gather_inputs", lambda: { - "settings": {}, "integrations": [], "accounts": [], "endpoints": [], - }) - - def slow(name): - def _fn(*_a, **_k): - time.sleep(0.6) - return {"name": name, "status": sh.OK, "detail": "", "meta": {}} - return _fn - - monkeypatch.setattr(sh, "searxng_health", slow("searxng")) - monkeypatch.setattr(sh, "ntfy_health", slow("ntfy")) - monkeypatch.setattr(sh, "email_health", slow("email")) - monkeypatch.setattr(sh, "providers_health", slow("providers")) - - t0 = time.monotonic() - out = asyncio.run(sh.collect_service_health(None, None)) - elapsed = time.monotonic() - t0 - assert elapsed < 1.5, f"subsystems not concurrent: took {elapsed:.1f}s" - assert {s["name"] for s in out["services"]} == { - "chromadb", "searxng", "ntfy", "email", "providers"} - - -def test_collect_aggregate_deadline_yields_controlled_result(monkeypatch): - # If the gather overruns the aggregate ceiling, the response is still a - # controlled {overall, services, timestamp} with each network subsystem - # marked down/timeout — never a hang or a raised exception. - import asyncio - import time - monkeypatch.setattr(sh, "_AGGREGATE_DEADLINE", 0.5) - monkeypatch.setattr(sh, "_SUBSYSTEM_DEADLINE", 0.4) - monkeypatch.setattr(sh, "_gather_inputs", lambda: { - "settings": {}, "integrations": [], "accounts": [], "endpoints": [], - }) - - async def _slow_gather(*coros, **_k): - for c in coros: # close unawaited coros to avoid warnings - close = getattr(c, "close", None) - if close: - close() - await asyncio.sleep(5) - - # Force the outer wait_for to trip by making gather itself slow. - monkeypatch.setattr(sh.asyncio, "gather", _slow_gather) - t0 = time.monotonic() - out = asyncio.run(sh.collect_service_health(None, None)) - elapsed = time.monotonic() - t0 - assert elapsed < 2, f"aggregate deadline did not bound: {elapsed:.1f}s" - assert set(out) == {"overall", "services", "timestamp"} - net = [s for s in out["services"] if s["name"] != "chromadb"] - assert all(s["status"] == sh.DOWN and s["meta"].get("error") == "timeout" - for s in net) diff --git a/tests/test_service_health_chromadb.py b/tests/test_service_health_chromadb.py new file mode 100644 index 000000000..290d1f986 --- /dev/null +++ b/tests/test_service_health_chromadb.py @@ -0,0 +1,37 @@ +"""Tests for chromadb_health — ok/degraded/down/disabled classification.""" +import pytest + +from src import service_health as sh + + +class _Store: + def __init__(self, healthy): + self.healthy = healthy + + +def test_chromadb_both_healthy_ok(): + s = sh.chromadb_health(_Store(True), _Store(True)) + assert s["status"] == sh.OK + assert s["meta"] == {"rag": True, "memory": True} + + +def test_chromadb_one_down_degraded(): + s = sh.chromadb_health(_Store(True), _Store(False)) + assert s["status"] == sh.DEGRADED + + +def test_chromadb_both_unhealthy_down(): + s = sh.chromadb_health(_Store(False), _Store(False)) + assert s["status"] == sh.DOWN + + +def test_chromadb_both_absent_disabled(): + s = sh.chromadb_health(None, None) + assert s["status"] == sh.DISABLED + + +def test_chromadb_one_absent_one_healthy_ok(): + # An absent store is not a failure; the present one being healthy is ok. + s = sh.chromadb_health(_Store(True), None) + assert s["status"] == sh.OK + assert s["meta"]["memory"] is None diff --git a/tests/test_service_health_collect.py b/tests/test_service_health_collect.py new file mode 100644 index 000000000..40e2d6f60 --- /dev/null +++ b/tests/test_service_health_collect.py @@ -0,0 +1,139 @@ +"""Tests for rollup logic, aggregate collection, and shared utility helpers (_safe_url, _classify_error).""" +import pytest + +from src import service_health as sh + + +class _Store: + def __init__(self, healthy): + self.healthy = healthy + + +# ── rollup ── + +def test_rollup_picks_worst_non_disabled(): + services = [ + {"status": sh.OK}, {"status": sh.DISABLED}, + {"status": sh.DEGRADED}, {"status": sh.OK}, + ] + assert sh._rollup(services) == sh.DEGRADED + + +def test_rollup_down_beats_degraded(): + assert sh._rollup([{"status": sh.DEGRADED}, {"status": sh.DOWN}]) == sh.DOWN + + +def test_rollup_all_disabled_is_ok(): + assert sh._rollup([{"status": sh.DISABLED}, {"status": sh.DISABLED}]) == sh.OK + + +# ── collect_service_health (async aggregate) ── + +def test_collect_service_health_shape(monkeypatch): + import asyncio + + # Avoid touching real data sources / network. + monkeypatch.setattr(sh, "_gather_inputs", lambda: { + "settings": {"search_provider": "disabled"}, + "integrations": [], + "accounts": [], + "endpoints": [], + }) + out = asyncio.run(sh.collect_service_health(_Store(True), _Store(True))) + assert set(out) == {"overall", "services", "timestamp"} + names = {s["name"] for s in out["services"]} + assert names == {"chromadb", "searxng", "ntfy", "email", "providers"} + # Chroma healthy, everything else disabled → overall ok. + assert out["overall"] == sh.OK + + +# ── _safe_url: strip userinfo / query / fragment ── + +@pytest.mark.parametrize("raw,expected", [ + ("http://user:pass@host:8080/path?api_key=secret#frag", "http://host:8080/path"), + ("https://admin:hunter2@searx.example.com/", "https://searx.example.com"), + ("http://ntfy.local:80?token=abc", "http://ntfy.local:80"), + ("host:8080", "host:8080"), + ("", ""), + (None, ""), +]) +def test_safe_url_strips_secrets(raw, expected): + out = sh._safe_url(raw) + assert out == expected + for bad in ("pass", "secret", "hunter2", "abc", "token", "@"): + if raw and bad in raw and bad not in expected: + assert bad not in out + + +# ── _classify_error: controlled categories, never raw text ── + +def test_classify_error_categories(): + import socket + assert sh._classify_error(TimeoutError()) == "timeout" + assert sh._classify_error(socket.timeout()) == "timeout" + assert sh._classify_error(socket.gaierror()) == "dns_error" + assert sh._classify_error(ConnectionRefusedError()) == "connection_refused" + assert sh._classify_error(OSError("boom")) == "network_error" + assert sh._classify_error(ValueError("x")) == "error" + + +# ── Concurrent collection and aggregate deadline ── + +def test_collect_runs_subsystems_concurrently(monkeypatch): + # The aggregate is bounded by running the (internally-bounded) subsystems + # concurrently, so total wall-clock ≈ max(subsystem), not the sum. Each of + # the four network subsystems here sleeps ~0.6s; sequential would be ~2.4s. + import asyncio + import time + monkeypatch.setattr(sh, "_gather_inputs", lambda: { + "settings": {}, "integrations": [], "accounts": [], "endpoints": [], + }) + + def slow(name): + def _fn(*_a, **_k): + time.sleep(0.6) + return {"name": name, "status": sh.OK, "detail": "", "meta": {}} + return _fn + + monkeypatch.setattr(sh, "searxng_health", slow("searxng")) + monkeypatch.setattr(sh, "ntfy_health", slow("ntfy")) + monkeypatch.setattr(sh, "email_health", slow("email")) + monkeypatch.setattr(sh, "providers_health", slow("providers")) + + t0 = time.monotonic() + out = asyncio.run(sh.collect_service_health(None, None)) + elapsed = time.monotonic() - t0 + assert elapsed < 1.5, f"subsystems not concurrent: took {elapsed:.1f}s" + assert {s["name"] for s in out["services"]} == { + "chromadb", "searxng", "ntfy", "email", "providers"} + + +def test_collect_aggregate_deadline_yields_controlled_result(monkeypatch): + # If the gather overruns the aggregate ceiling, the response is still a + # controlled {overall, services, timestamp} with each network subsystem + # marked down/timeout — never a hang or a raised exception. + import asyncio + import time + monkeypatch.setattr(sh, "_AGGREGATE_DEADLINE", 0.5) + monkeypatch.setattr(sh, "_SUBSYSTEM_DEADLINE", 0.4) + monkeypatch.setattr(sh, "_gather_inputs", lambda: { + "settings": {}, "integrations": [], "accounts": [], "endpoints": [], + }) + + async def _slow_gather(*coros, **_k): + for c in coros: # close unawaited coros to avoid warnings + close = getattr(c, "close", None) + if close: + close() + await asyncio.sleep(5) + + # Force the outer wait_for to trip by making gather itself slow. + monkeypatch.setattr(sh.asyncio, "gather", _slow_gather) + t0 = time.monotonic() + out = asyncio.run(sh.collect_service_health(None, None)) + elapsed = time.monotonic() - t0 + assert elapsed < 2, f"aggregate deadline did not bound: {elapsed:.1f}s" + assert set(out) == {"overall", "services", "timestamp"} + net = [s for s in out["services"] if s["name"] != "chromadb"] + assert all(s["status"] == sh.DOWN and s["meta"].get("error") == "timeout" + for s in net) diff --git a/tests/test_service_health_email.py b/tests/test_service_health_email.py new file mode 100644 index 000000000..5ae490b1c --- /dev/null +++ b/tests/test_service_health_email.py @@ -0,0 +1,80 @@ +"""Tests for email_health — probe logic, status classification, sanitization, and bounded timeout.""" +import pytest + +from src import service_health as sh + + +def _raise(*_a, **_k): + raise RuntimeError("connection refused") + + +def _acct(name, host="imap.example.com"): + return {"account_id": name, "account_name": name, "imap_host": host, + "imap_password": "hunter2"} + + +class _Conn: + def logout(self): + pass + + +def test_email_disabled_without_accounts(): + assert sh.email_health([])["status"] == sh.DISABLED + + +def test_email_ok_all_connect(): + s = sh.email_health([_acct("a"), _acct("b")], connect=lambda _id: _Conn()) + assert s["status"] == sh.OK + + +def test_email_degraded_some_fail(): + def connect(account_id): + if account_id == "bad": + raise RuntimeError("auth failed") + return _Conn() + + s = sh.email_health([_acct("good"), _acct("bad")], connect=connect) + assert s["status"] == sh.DEGRADED + + +def test_email_down_all_fail(): + s = sh.email_health([_acct("a")], connect=_raise) + assert s["status"] == sh.DOWN + + +def test_email_account_without_host_marked_failed(): + s = sh.email_health([_acct("a", host="")], connect=lambda _id: _Conn()) + assert s["status"] == sh.DOWN + + +def test_email_meta_never_leaks_password(): + s = sh.email_health([_acct("a")], connect=lambda _id: _Conn()) + assert "hunter2" not in repr(s) + + +def test_email_connect_exception_maps_to_category(): + def boom(account_id): + raise RuntimeError("login failed for user bob with password hunter2") + s = sh.email_health([_acct("a")], connect=boom) + assert s["status"] == sh.DOWN + assert s["meta"]["accounts"][0]["error"] == "error" + assert "hunter2" not in repr(s) + + +def test_email_bounded_marks_slow_as_timeout(monkeypatch): + import time + monkeypatch.setattr(sh, "_FANOUT_BUDGET", 1) + + def connect(account_id): + if account_id == "slow": + time.sleep(10) + return _Conn() + + accts = [_acct("fast"), _acct("slow")] + accts[1]["account_id"] = "slow" + t0 = time.monotonic() + out = sh.email_health(accts, connect=connect) + elapsed = time.monotonic() - t0 + assert elapsed < 4, f"email_health not bounded: took {elapsed:.1f}s" + by = {a["name"]: a for a in out["meta"]["accounts"]} + assert by["slow"]["error"] == "timeout" diff --git a/tests/test_service_health_ntfy.py b/tests/test_service_health_ntfy.py new file mode 100644 index 000000000..f8820f2ac --- /dev/null +++ b/tests/test_service_health_ntfy.py @@ -0,0 +1,62 @@ +"""Tests for ntfy_health — probe logic, status classification, and sanitization.""" +import types + +import pytest + +from src import service_health as sh + + +def _resp(status_code): + return types.SimpleNamespace(status_code=status_code) + + +def _raise(*_a, **_k): + raise RuntimeError("connection refused") + + +def _ntfy_intg(): + return [{"preset": "ntfy", "enabled": True, "base_url": "http://ntfy:80"}] + + +def test_ntfy_disabled_without_integration(): + s = sh.ntfy_health([], {"reminder_channel": "ntfy"}) + assert s["status"] == sh.DISABLED + + +def test_ntfy_ok(): + s = sh.ntfy_health(_ntfy_intg(), {"reminder_channel": "ntfy"}, + http_get=lambda url, timeout: _resp(200)) + assert s["status"] == sh.OK + assert s["meta"]["base"] == "http://ntfy:80" + + +def test_ntfy_probes_v1_health_not_a_topic(): + seen = {} + + def getter(url, timeout): + seen["url"] = url + return _resp(200) + + sh.ntfy_health(_ntfy_intg(), {"reminder_channel": "ntfy"}, http_get=getter) + # Non-intrusive: hits /v1/health, never publishes to a topic. + assert seen["url"].endswith("/v1/health") + + +def test_ntfy_down_on_exception(): + s = sh.ntfy_health(_ntfy_intg(), {"reminder_channel": "ntfy"}, + http_get=_raise) + assert s["status"] == sh.DOWN + + +def test_ntfy_meta_redacts_userinfo_in_base(): + intg = [{"preset": "ntfy", "enabled": True, + "base_url": "https://user:topsecret@ntfy.example.com"}] + seen = {} + + def getter(url, timeout): + seen["url"] = url # the probe itself may keep credentials + return _resp(200) + + s = sh.ntfy_health(intg, {"reminder_channel": "ntfy"}, http_get=getter) + assert s["meta"]["base"] == "https://ntfy.example.com" + assert "topsecret" not in repr(s) diff --git a/tests/test_service_health_providers.py b/tests/test_service_health_providers.py new file mode 100644 index 000000000..ad2d72794 --- /dev/null +++ b/tests/test_service_health_providers.py @@ -0,0 +1,100 @@ +"""Tests for providers_health — probe logic, status classification, sanitization, and bounded timeout.""" +import pytest + +from src import service_health as sh + + +def _raise(*_a, **_k): + raise RuntimeError("connection refused") + + +def _ep(name): + return {"name": name, "base_url": f"http://{name}:8000/v1", "api_key": "sk-secret"} + + +def test_providers_disabled_without_endpoints(): + assert sh.providers_health([])["status"] == sh.DISABLED + + +def test_providers_ok_all_reachable(): + s = sh.providers_health([_ep("a")], + probe=lambda base, key, timeout: ["m1", "m2"]) + assert s["status"] == sh.OK + assert s["meta"]["endpoints"][0]["model_count"] == 2 + + +def test_providers_degraded_some_empty(): + def probe(base, key, timeout): + return ["m1"] if "good" in base else [] + + s = sh.providers_health([_ep("good"), _ep("bad")], probe=probe) + assert s["status"] == sh.DEGRADED + + +def test_providers_down_all_fail(): + s = sh.providers_health([_ep("a")], probe=_raise) + assert s["status"] == sh.DOWN + + +def test_providers_meta_never_leaks_api_key(): + s = sh.providers_health([_ep("a")], + probe=lambda base, key, timeout: ["m1"]) + assert "sk-secret" not in repr(s) + + +def test_providers_name_fallback_is_sanitized(): + # No display name → falls back to the base_url, which must be sanitized. + ep = {"base_url": "http://user:k3y@prov.local:9000/v1?api_key=zzz", "api_key": "sk-x"} + s = sh.providers_health([ep], probe=lambda b, k, t: ["m1"]) + entry = s["meta"]["endpoints"][0] + assert entry["name"] == "http://prov.local:9000/v1" + assert "k3y" not in repr(s) and "zzz" not in repr(s) and "sk-x" not in repr(s) + + +def test_providers_probe_exception_maps_to_category(): + def boom(base, key, timeout): + raise RuntimeError(f"500 from {base} with key {key}") # would leak base+key + s = sh.providers_health([_ep("a")], probe=boom) + assert s["status"] == sh.DOWN + assert s["meta"]["endpoints"][0]["error"] == "error" + assert "sk-secret" not in repr(s) and "http://a" not in repr(s) + + +def test_providers_bounded_marks_slow_as_timeout(monkeypatch): + import time + monkeypatch.setattr(sh, "_FANOUT_BUDGET", 1) + + def probe(base, key, timeout): + if "slow" in base: + time.sleep(10) # would blow the budget if unbounded + return ["m1"] + + eps = [{"name": "fast", "base_url": "http://fast", "api_key": "k"}, + {"name": "slow", "base_url": "http://slow", "api_key": "k"}] + t0 = time.monotonic() + out = sh.providers_health(eps, probe=probe) + elapsed = time.monotonic() - t0 + assert elapsed < 4, f"providers_health not bounded: took {elapsed:.1f}s" + by = {e["name"]: e for e in out["meta"]["endpoints"]} + assert by["fast"]["ok"] is True + assert by["slow"]["ok"] is False and by["slow"]["error"] == "timeout" + assert out["status"] == sh.DEGRADED + + +def test_providers_bounded_with_many_slow_endpoints(monkeypatch): + import time + monkeypatch.setattr(sh, "_FANOUT_BUDGET", 1) + + def probe(base, key, timeout): + time.sleep(10) + return ["m1"] + + eps = [{"name": f"ep{i}", "base_url": f"http://ep{i}", "api_key": "k"} + for i in range(25)] + t0 = time.monotonic() + out = sh.providers_health(eps, probe=probe) + elapsed = time.monotonic() - t0 + # 25 endpoints * sleep would be huge if sequential; bounded keeps it ~budget. + assert elapsed < 4, f"not bounded with many endpoints: {elapsed:.1f}s" + assert out["status"] == sh.DOWN + assert all(e["error"] == "timeout" for e in out["meta"]["endpoints"]) diff --git a/tests/test_service_health_search.py b/tests/test_service_health_search.py new file mode 100644 index 000000000..56553808a --- /dev/null +++ b/tests/test_service_health_search.py @@ -0,0 +1,79 @@ +"""Tests for searxng_health — probe logic, status classification, and sanitization.""" +import types + +import pytest + +from src import service_health as sh + + +def _resp(status_code): + return types.SimpleNamespace(status_code=status_code) + + +def _raise(*_a, **_k): + raise RuntimeError("connection refused") + + +def test_searxng_disabled_when_other_provider(): + s = sh.searxng_health({"search_provider": "brave"}) + assert s["status"] == sh.DISABLED + + +def test_searxng_ok_on_healthz(): + s = sh.searxng_health( + {"search_provider": "searxng", "search_url": "http://sx:8080"}, + http_get=lambda url, timeout: _resp(200), + ) + assert s["status"] == sh.OK + assert s["meta"]["probed"] == "/healthz" + + +def test_searxng_ok_on_root_fallback(): + def getter(url, timeout): + return _resp(404) if url.endswith("/healthz") else _resp(200) + + s = sh.searxng_health( + {"search_provider": "searxng", "search_url": "http://sx:8080"}, + http_get=getter, + ) + assert s["status"] == sh.OK + assert s["meta"]["probed"] == "/" + + +def test_searxng_down_on_exception(): + s = sh.searxng_health( + {"search_provider": "searxng", "search_url": "http://sx:8080"}, + http_get=_raise, + ) + assert s["status"] == sh.DOWN + + +def test_searxng_down_on_5xx(): + s = sh.searxng_health( + {"search_provider": "searxng", "search_url": "http://sx:8080"}, + http_get=lambda url, timeout: _resp(502), + ) + assert s["status"] == sh.DOWN + + +def test_searxng_meta_redacts_instance_url(): + s = sh.searxng_health( + {"search_provider": "searxng", + "search_url": "http://user:s3cr3t@searx.local:8080/?token=zzz"}, + http_get=lambda url, timeout: _resp(200), + ) + blob = repr(s) + assert "s3cr3t" not in blob and "zzz" not in blob and "user:" not in blob + assert s["meta"]["instance"] == "http://searx.local:8080" + + +def test_searxng_down_uses_error_category_not_raw_exception(): + def boom(url, timeout): + raise RuntimeError("failed connecting to http://user:pw@searx.local secret-token") + s = sh.searxng_health( + {"search_provider": "searxng", "search_url": "http://searx.local"}, + http_get=boom, + ) + assert s["status"] == sh.DOWN + assert s["meta"]["error"] == "error" # controlled category token + assert "secret-token" not in repr(s) and "pw@" not in repr(s) From 1f6dc805250415d330d900af77b8749e254c76d5 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:17:28 +0100 Subject: [PATCH 15/32] ci: add focused test guidance signal (#4982) * ci: add focused test guidance signal * ci: diff focused guidance from merge base --- .github/scripts/focused_test_guidance.py | 124 +++++++++++++++++++ .github/workflows/ci.yml | 54 +++++++++ tests/test_focused_test_guidance.py | 148 +++++++++++++++++++++++ 3 files changed, 326 insertions(+) create mode 100644 .github/scripts/focused_test_guidance.py create mode 100644 tests/test_focused_test_guidance.py diff --git a/.github/scripts/focused_test_guidance.py b/.github/scripts/focused_test_guidance.py new file mode 100644 index 000000000..1426d35fa --- /dev/null +++ b/.github/scripts/focused_test_guidance.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Report focused pytest guidance for changed paths under tests/.""" + +from __future__ import annotations + +import argparse +import os +import shlex +import subprocess +import sys +from collections.abc import Iterable +from pathlib import PurePosixPath + + +def parse_paths(raw_paths: bytes) -> list[str]: + """Decode the NUL-delimited output of ``git diff --name-only -z``.""" + return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path] + + +def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]: + """Return changed ``tests/`` paths using GitHub PR three-dot semantics. + + GitHub PR changed files are based on the merge base and the PR head, not a + direct endpoint diff between the current base branch tip and the PR head. + Using the direct endpoint diff can include files changed only on the base + branch when the PR branch is stale. + """ + merge_base = subprocess.check_output( + ["git", "merge-base", base_sha, head_sha], + stderr=subprocess.DEVNULL, + ).strip() + raw_paths = subprocess.check_output( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMRT", + "-z", + os.fsdecode(merge_base), + head_sha, + "--", + "tests/", + ], + ) + return parse_paths(raw_paths) + + +def select_test_paths(paths: Iterable[str]) -> list[str]: + """Return unique, repository-relative paths contained by tests/.""" + selected: set[str] = set() + for raw_path in paths: + path = PurePosixPath(raw_path) + if path.is_absolute() or ".." in path.parts: + continue + parts = tuple(part for part in path.parts if part != ".") + if len(parts) >= 2 and parts[0] == "tests": + selected.add(PurePosixPath(*parts).as_posix()) + return sorted(selected) + + +def is_pytest_file(path: str) -> bool: + """Return whether a changed path follows this repository's pytest naming.""" + name = PurePosixPath(path).name + return name.endswith(".py") and ( + name.startswith("test_") or name.endswith("_test.py") + ) + + +def pytest_command(paths: Iterable[str]) -> str: + """Build a copyable pytest command for changed runnable test files.""" + command = ["python3", "-m", "pytest", "-q", *paths] + return shlex.join(command) + + +def format_report(paths: Iterable[str]) -> str: + """Format focused guidance for CI logs and the workflow summary.""" + changed_paths = select_test_paths(paths) + runnable_paths = [path for path in changed_paths if is_pytest_file(path)] + lines = ["## Focused test guidance (report-only)", ""] + if not changed_paths: + lines.append("No changed paths under `tests/`.") + else: + lines.extend(["Changed paths under `tests/`:", ""]) + lines.extend(f"- `{path}`" for path in changed_paths) + lines.extend(["", "Suggested focused validation:", ""]) + if runnable_paths: + lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```") + else: + lines.append("No directly runnable pytest files changed.") + lines.extend( + [ + "", + "This guidance does not infer tests from source changes. " + "Existing blocking CI remains the source of truth.", + ] + ) + return "\n".join(lines) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Report focused pytest guidance for changed tests/ paths.", + ) + parser.add_argument("--base-sha", help="Pull request base commit SHA.") + parser.add_argument("--head-sha", help="Pull request head commit SHA.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + if bool(args.base_sha) != bool(args.head_sha): + raise SystemExit("--base-sha and --head-sha must be provided together") + + if args.base_sha and args.head_sha: + paths = changed_paths_from_merge_base(args.base_sha, args.head_sha) + else: + paths = parse_paths(sys.stdin.buffer.read()) + + print(format_report(paths)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 787bd9dea..f7d3659e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,60 @@ concurrency: cancel-in-progress: true jobs: + focused-test-guidance: + name: Focused test guidance (report-only) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: Report changed test paths + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + report_file="$RUNNER_TEMP/focused-test-guidance.md" + publish_report() { + cat "$report_file" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true + fi + return 0 + } + + report_unavailable() { + { + printf '%s\n\n' '## Focused test guidance unavailable (report-only)' + printf '%s\n\n' "$1" + printf '%s\n' 'Existing blocking CI remains the source of truth.' + } > "$report_file" + publish_report + exit 0 + } + + if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then + report_unavailable "Pull request base/head metadata is missing." + fi + + if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then + report_unavailable "The pull request base commit is unavailable locally." + fi + + if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then + report_unavailable "The pull request head commit is unavailable locally." + fi + + if ! python3 .github/scripts/focused_test_guidance.py \ + --base-sha "$BASE_SHA" \ + --head-sha "$HEAD_SHA" > "$report_file"; then + report_unavailable "The focused test guidance helper could not produce a report." + fi + + publish_report + python-syntax: name: Python syntax (compileall) runs-on: ubuntu-latest diff --git a/tests/test_focused_test_guidance.py b/tests/test_focused_test_guidance.py new file mode 100644 index 000000000..adf46f9e4 --- /dev/null +++ b/tests/test_focused_test_guidance.py @@ -0,0 +1,148 @@ +"""Tests for the report-only changed-test guidance helper.""" + +from __future__ import annotations + +import importlib.util +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_PATH = ROOT / ".github" / "scripts" / "focused_test_guidance.py" + + +def _load_helper(): + spec = importlib.util.spec_from_file_location("focused_test_guidance", SCRIPT_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +guidance = _load_helper() + + +def test_parse_paths_supports_nul_delimited_git_output(): + raw_paths = b"tests/test_alpha.py\0tests/path with spaces/test_beta.py\0" + + assert guidance.parse_paths(raw_paths) == [ + "tests/test_alpha.py", + "tests/path with spaces/test_beta.py", + ] + + +def test_select_test_paths_ignores_paths_outside_tests(): + paths = [ + "src/test_alpha.py", + "tests/test_beta.py", + "./tests/unit/example_test.py", + "tests/../src/test_gamma.py", + ] + + assert guidance.select_test_paths(paths) == [ + "tests/test_beta.py", + "tests/unit/example_test.py", + ] + + +def test_select_test_paths_deduplicates_paths(): + paths = ["tests/test_alpha.py", "./tests/test_alpha.py"] + + assert guidance.select_test_paths(paths) == ["tests/test_alpha.py"] + + +def test_format_report_builds_command_for_changed_python_test(): + report = guidance.format_report(["tests/test_beta.py"]) + + assert "- `tests/test_beta.py`" in report + assert "python3 -m pytest -q tests/test_beta.py" in report + assert "does not infer tests from source changes" in report + assert "Existing blocking CI remains the source of truth" in report + + +def test_format_report_lists_changed_non_python_test_path_without_command(): + report = guidance.format_report(["tests/README.md"]) + + assert "- `tests/README.md`" in report + assert "No directly runnable pytest files changed." in report + assert "python3 -m pytest" not in report + + +def test_format_report_ignores_src_path(): + report = guidance.format_report(["src/test_ignored.py"]) + + assert "src/test_ignored.py" not in report + assert "No changed paths under `tests/`." in report + + +def test_format_report_shell_quotes_path_with_spaces(): + report = guidance.format_report(["tests/path with spaces/test_beta.py"]) + + assert "- `tests/path with spaces/test_beta.py`" in report + assert ( + "python3 -m pytest -q 'tests/path with spaces/test_beta.py'" + ) in report + + +def test_format_report_handles_no_changed_test_paths(): + report = guidance.format_report([]) + + assert "No changed paths under `tests/`." in report + assert "No directly runnable pytest files changed." in report + +def _git(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() + + +def _write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def test_changed_paths_from_merge_base_excludes_base_only_test_changes(tmp_path, monkeypatch): + repo = tmp_path / "repo" + repo.mkdir() + + _git(repo, "init") + _git(repo, "config", "user.email", "ci@example.test") + _git(repo, "config", "user.name", "CI Test") + + _write(repo / "tests/test_shared.py", "def test_shared():\n assert True\n") + _git(repo, "add", "tests/test_shared.py") + _git(repo, "commit", "-m", "base") + ancestor = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "feature") + _write(repo / "tests/test_pr_delta.py", "def test_pr_delta():\n assert True\n") + _git(repo, "add", "tests/test_pr_delta.py") + _git(repo, "commit", "-m", "add pr test") + head_sha = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "dev", ancestor) + _write(repo / "tests/test_shared.py", "def test_shared():\n assert 1 == 1\n") + _git(repo, "add", "tests/test_shared.py") + _git(repo, "commit", "-m", "base-only test change") + base_sha = _git(repo, "rev-parse", "HEAD") + + endpoint_paths = guidance.parse_paths( + subprocess.check_output( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMRT", + "-z", + base_sha, + head_sha, + "--", + "tests/", + ], + cwd=repo, + ) + ) + assert "tests/test_shared.py" in endpoint_paths + + monkeypatch.chdir(repo) + assert guidance.changed_paths_from_merge_base(base_sha, head_sha) == [ + "tests/test_pr_delta.py" + ] From d3ab478ef1783381024328d06f3895c90b5b626e Mon Sep 17 00:00:00 2001 From: harshit-ojha0324 <92013456+harshit-ojha0324@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:26:36 -0400 Subject: [PATCH 16/32] fix(integrations): don't append a trailing slash when api_call path is '/' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _join_integration_url built urljoin(base + '/', '') for a bare '/' path — the minimum execute_api_call accepts — so every request against a POST-to-base integration went to base_url + '/'. Discord webhook URLs 404 ('Unknown Webhook') on the trailing-slash variant, which made the integration look broken even though the stored base URL was correct. Resolve a bare '/' (or empty) path to the base URL itself and keep all other paths joining exactly as before, including deliberate trailing slashes inside non-empty paths (linkding /api/tags/, Home Assistant /api/). The reminder webhook sender and the discord_webhook connectivity test already posted to the bare base URL; execute_api_call was the remaining path that re-added the slash. Fixes #5138 --- src/integrations.py | 9 ++- tests/test_integrations_url_join.py | 117 ++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/test_integrations_url_join.py diff --git a/src/integrations.py b/src/integrations.py index 3b2b88859..157cf922d 100644 --- a/src/integrations.py +++ b/src/integrations.py @@ -216,7 +216,14 @@ def _normalize_integration_base_url(base_url: Any) -> str: def _join_integration_url(base_url: str, path: str) -> str: - return urljoin(base_url.rstrip("/") + "/", path.lstrip("/")) + base = base_url.rstrip("/") + rel = path.lstrip("/") + if not rel: + # A bare "/" must resolve to the base URL itself, not base + "/". + # POST-to-base integrations (e.g. Discord webhooks) 404 on the + # trailing-slash variant of their URL. + return base + return urljoin(base + "/", rel) def load_integrations() -> List[Dict[str, Any]]: diff --git a/tests/test_integrations_url_join.py b/tests/test_integrations_url_join.py new file mode 100644 index 000000000..9a6359038 --- /dev/null +++ b/tests/test_integrations_url_join.py @@ -0,0 +1,117 @@ +"""Tests for integration URL construction in execute_api_call. + +Covers the trailing-slash regression from #5138: a bare "/" path must +resolve to the base URL itself, not base + "/". Discord webhook URLs +404 on the trailing-slash variant, so api_call against a +POST-to-base integration silently failed. +""" +import sys +import types +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Minimal stubs so src.integrations can be imported without heavy deps +# --------------------------------------------------------------------------- + +for mod_name in ("core", "core.atomic_io", "core.platform_compat"): + if mod_name not in sys.modules: + sys.modules[mod_name] = types.ModuleType(mod_name) + +core_atomic = sys.modules["core.atomic_io"] +if not hasattr(core_atomic, "atomic_write_json"): + core_atomic.atomic_write_json = lambda *a, **kw: None # type: ignore + +core_compat = sys.modules["core.platform_compat"] +if not hasattr(core_compat, "safe_chmod"): + core_compat.safe_chmod = lambda *a, **kw: None # type: ignore + +if "src.secret_storage" not in sys.modules: + stub = types.ModuleType("src.secret_storage") + stub.encrypt = lambda s: s # type: ignore + stub.decrypt = lambda s: s # type: ignore + stub.is_encrypted = lambda s: False # type: ignore + sys.modules["src.secret_storage"] = stub + +if "src.constants" not in sys.modules: + stub_c = types.ModuleType("src.constants") + stub_c.DATA_DIR = "/tmp" # type: ignore + stub_c.INTEGRATIONS_FILE = "/tmp/integrations_test.json" # type: ignore + stub_c.SETTINGS_FILE = "/tmp/settings_test.json" # type: ignore + sys.modules["src.constants"] = stub_c + +from src import integrations # noqa: E402 + + +# --------------------------------------------------------------------------- +# _join_integration_url unit tests +# --------------------------------------------------------------------------- + +WEBHOOK_BASE = "https://discord.com/api/webhooks/123/tokentokentoken" + + +@pytest.mark.parametrize( + "base,path,expected", + [ + # Bare "/" (the minimum path execute_api_call accepts) must not + # grow a trailing slash — Discord webhooks 404 on it (#5138). + (WEBHOOK_BASE, "/", WEBHOOK_BASE), + (WEBHOOK_BASE + "/", "/", WEBHOOK_BASE), + (WEBHOOK_BASE, "", WEBHOOK_BASE), + # Normal paths keep joining exactly as before. + ("http://api.example.com", "/items", "http://api.example.com/items"), + ("http://api.example.com/", "/items", "http://api.example.com/items"), + ("http://host/base", "/v1/me", "http://host/base/v1/me"), + # A deliberate trailing slash inside a non-empty path is preserved + # (e.g. linkding's /api/tags/, Home Assistant's /api/). + ("http://host", "/api/tags/", "http://host/api/tags/"), + ("http://host", "/api/", "http://host/api/"), + ], +) +def test_join_integration_url(base, path, expected): + assert integrations._join_integration_url(base, path) == expected + + +# --------------------------------------------------------------------------- +# Behavioral test through execute_api_call +# --------------------------------------------------------------------------- + +DISCORD_INTEGRATION = { + "id": "discord_test", + "name": "Discord Webhook", + "enabled": True, + "base_url": WEBHOOK_BASE, + "auth_type": "none", + "api_key": "", + "auth_header": "", + "auth_param": "", + "description": "", + "preset": "discord_webhook", +} + + +@pytest.mark.asyncio +async def test_api_call_root_path_has_no_trailing_slash(): + mock_resp = MagicMock() + mock_resp.status_code = 204 + mock_resp.headers = {"content-type": "text/plain"} + mock_resp.text = "" + + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.request = AsyncMock(return_value=mock_resp) + + with ( + patch.object(integrations, "_find_integration", return_value=DISCORD_INTEGRATION), + patch("httpx.AsyncClient", return_value=mock_client), + ): + result = await integrations.execute_api_call( + "discord_test", "POST", "/", body={"content": "test"} + ) + + assert result.get("exit_code") == 0 + requested_url = mock_client.request.call_args.args[1] + assert requested_url == WEBHOOK_BASE + assert not requested_url.endswith("/") From 43ead1a0eb6c8acf449e8e214c6c5128c4798b38 Mon Sep 17 00:00:00 2001 From: ashvinctrl Date: Sat, 4 Jul 2026 14:13:04 +0530 Subject: [PATCH 17/32] fix(security): scope send_to_session to an exact session owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit send_to_session let an authenticated caller reach a null-owner session. The owner gate was `if owner and sess.owner and sess.owner != owner`, so a target whose owner is None (legacy rows, or a session created while auth was off) skipped the check and was read/written by any authenticated user. list_sessions (get_sessions_for_user) and manage_session already exclude null-owner sessions from an authenticated caller via an exact owner match, so this path was the lone inconsistency — the same class of gap the calendar owner=None fix closed. Require an exact owner match: `if owner and sess.owner != owner`. Auth-off (no owner) is unchanged, an exact-owner match still passes, and both another user's session and a null-owner session are now not-found. Adds a regression test that an authenticated caller cannot read the transcript of or write into a null-owner session while single-user access still works. --- src/agent_tools/session_tools.py | 8 +++-- tests/test_session_tools_registry.py | 49 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/agent_tools/session_tools.py b/src/agent_tools/session_tools.py index 28e48d35d..d714453c6 100644 --- a/src/agent_tools/session_tools.py +++ b/src/agent_tools/session_tools.py @@ -184,8 +184,12 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner: if not sess: return {"error": f"Session '{target_sid}' not found"} - # Owner-scope: reject access to another user's session - if owner and getattr(sess, "owner", None) and sess.owner != owner: + # Owner-scope: reject access to another user's session. When the caller is + # authenticated, a null-owner (legacy / auth-was-off) session is not theirs + # either — list_sessions (get_sessions_for_user) and manage_session already + # exclude those, so treating it as reachable here let an authenticated agent + # read/write a session the other tools hide. Require an exact owner match. + if owner and getattr(sess, "owner", None) != owner: return {"error": f"Session '{target_sid}' not found"} if not message: diff --git a/tests/test_session_tools_registry.py b/tests/test_session_tools_registry.py index 804cfdbdc..4f63f550f 100644 --- a/tests/test_session_tools_registry.py +++ b/tests/test_session_tools_registry.py @@ -137,6 +137,55 @@ def test_no_session_manager_is_handled(monkeypatch): assert "error" in res or "results" in res +class _FakeSession: + def __init__(self, owner, name, history): + self.owner = owner + self.name = name + self.endpoint_url = "http://x" + self.model = "fixture-tool-model" # offline path: returns transcript, no network + self._history = history + self.added = [] + + def get_context_messages(self): + return list(self._history) + + def add_message(self, m): + self.added.append(m) + + +class _FakeMgr: + def __init__(self, sessions): + self._s = sessions + + def get_session(self, sid): + return self._s.get(sid) + + +def test_send_to_session_blocks_null_owner_for_authenticated_caller(monkeypatch): + # An authenticated caller must not reach a null-owner (legacy / auth-was-off) + # session: list_sessions and manage_session already hide those, so this path + # was the inconsistency — it let an agent read/write a session the other + # tools exclude. Mirrors the calendar owner=None hardening. + null_sess = _FakeSession(None, "Secret", [{"role": "user", "content": "PIN 4321"}]) + bob_sess = _FakeSession("bob", "Bob", [{"role": "user", "content": "bob secret"}]) + monkeypatch.setattr(st, "get_session_manager", + lambda: _FakeMgr({"nsid": null_sess, "bsid": bob_sess})) + + # authenticated alice: null-owner session is not-found and its history is not leaked + r = asyncio.run(st.send_to_session("nsid\nhello", owner="alice")) + assert r.get("error", "").endswith("not found") + assert "4321" not in str(r) + assert null_sess.added == [] # nothing written into it either + + # authenticated alice still cannot reach another real user's session + r2 = asyncio.run(st.send_to_session("bsid\nhello", owner="alice")) + assert r2.get("error", "").endswith("not found") + + # auth disabled (no owner): single-user still reaches the null-owner session + r3 = asyncio.run(st.send_to_session("nsid\nhello", owner=None)) + assert r3.get("offline_transcript") is True + + def test_dispatched_via_registry_not_dispatch_ai_tool(): source = (Path(__file__).resolve().parent.parent / "src" / "tool_execution.py").read_text(encoding="utf-8") assert 'elif tool in ("create_session", "list_sessions", "send_to_session", "manage_session"):' in source From 6f6cb6ea8804e9940c5e17147330b3231c61e7b1 Mon Sep 17 00:00:00 2001 From: "Tal.Yuan" Date: Sat, 4 Jul 2026 19:36:35 +0800 Subject: [PATCH 18/32] refactor(routes): move history domain into routes/history/ subpackage (#5090) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice 2d of the route-domain reorganization (#4082/#4071, per specs/architecture-runtime-inventory.md §6.3). Moves history_routes.py into routes/history/, leaving a backward-compat sys.modules shim at the old path. Pure file reorganization, no behavior change. The shim uses sys.modules replacement (same pattern as the merged gallery #4903, research #4975, and memory #5007 slices) so that `import routes.history_routes`, `from routes.history_routes import X`, `importlib.import_module(...)`, and the `import ... as history_routes` + `monkeypatch.setattr(history_routes, ...)` pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py all operate on the same module object the application uses. The canonical module does NOT depend on the shim — routes/history/ history_routes.py imports only from core/, src/, and routes.session_routes (a sibling route module whose old import path stays valid via its own shim when session is migrated later). Three source-introspection test sites repointed to the new canonical path: - test_history_db_fallback_hidden.py - test_history_order_by_timestamp_regression.py - test_model_helper_owner_scope.py Adds tests/test_history_routes_shim.py to pin the sys.modules shim contract (legacy and canonical paths resolve to the same module object; monkeypatch via legacy alias reaches the canonical module). Verified: compileall clean; full suite 4351 passed, 3 skipped. --- app.py | 2 +- routes/history/__init__.py | 5 + routes/history/history_routes.py | 768 +++++++++++++++++ routes/history_routes.py | 777 +----------------- tests/test_history_db_fallback_hidden.py | 2 +- ...t_history_order_by_timestamp_regression.py | 2 +- tests/test_history_routes_shim.py | 43 + tests/test_model_helper_owner_scope.py | 2 +- 8 files changed, 833 insertions(+), 768 deletions(-) create mode 100644 routes/history/__init__.py create mode 100644 routes/history/history_routes.py create mode 100644 tests/test_history_routes_shim.py diff --git a/app.py b/app.py index ebd026807..8914893ab 100644 --- a/app.py +++ b/app.py @@ -645,7 +645,7 @@ from routes.research.research_routes import setup_research_routes app.include_router(setup_research_routes(research_handler, session_manager=session_manager)) # History -from routes.history_routes import setup_history_routes +from routes.history.history_routes import setup_history_routes app.include_router(setup_history_routes(session_manager)) # Search diff --git a/routes/history/__init__.py b/routes/history/__init__.py new file mode 100644 index 000000000..a9bd5627e --- /dev/null +++ b/routes/history/__init__.py @@ -0,0 +1,5 @@ +"""History route domain package (slice 2d, #4082/#4071). + +Contains history_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/history_routes.py re-exports from here. +""" diff --git a/routes/history/history_routes.py b/routes/history/history_routes.py new file mode 100644 index 000000000..2324ae286 --- /dev/null +++ b/routes/history/history_routes.py @@ -0,0 +1,768 @@ +"""History routes — session history, truncation, fork, conversation topics.""" + +import json +import uuid +import logging +import re +from typing import Dict, Any, Optional + +from fastapi import APIRouter, Request, HTTPException + +from core.models import ChatMessage +from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession +from src.topic_analyzer import analyze_topics +from routes.session_routes import ( + _message_role, + _message_text, + _reject_compact_during_active_run, + _verify_session_owner, +) + +logger = logging.getLogger(__name__) + +_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000 +_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+") + + +def _history_display_content(content: Any) -> Any: + """Return a lightweight browser-display copy of stored message content. + + Older multimodal user messages may be persisted as a JSON *string* + containing image_url blocks with inline base64 image bytes. Those bytes are + needed for model calls when the turn is first sent, but they should not be + sent back through /api/history every time the user opens the chat. The + attachment metadata already carries file ids/names for the UI cards. + """ + if isinstance(content, list): + text_parts = [] + omitted_media = 0 + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "text": + text = block.get("text") + if isinstance(text, str) and text: + text_parts.append(text) + elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}: + omitted_media += 1 + text = "\n".join(text_parts).strip() + if omitted_media and not text: + return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]" + return text + + if not isinstance(content, str): + return content + if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content: + return content + + stripped = content.lstrip() + if stripped.startswith("["): + try: + blocks = json.loads(content) + except (json.JSONDecodeError, TypeError, ValueError): + blocks = None + if isinstance(blocks, list): + text_parts = [] + for block in blocks: + if not isinstance(block, dict): + continue + if block.get("type") == "text": + text = block.get("text") + if isinstance(text, str) and text: + text_parts.append(text) + if text_parts: + return "\n".join(text_parts).strip() + + if "data:image/" in content: + return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content) + return content + + +def _merge_continue_rows_to_delete(db_messages, db1, db2): + """DB rows to delete when merging the last two assistant messages. + + Always the second assistant message (db2), plus ONLY the single + intervening "continue" user message (the one carrying "previous response + was interrupted") — matching the in-memory merge. The previous code + deleted the whole index range between the two assistant rows, destroying + any tool/system/user messages in between and desyncing the DB from the + in-memory history. + """ + to_delete = [db2] + i1 = next((i for i, m in enumerate(db_messages) if m is db1), None) + i2 = next((i for i, m in enumerate(db_messages) if m is db2), None) + if i1 is not None and i2 is not None and i2 - 1 > i1: + between = db_messages[i2 - 1] + if getattr(between, "role", "") == "user" and "previous response was interrupted" in (getattr(between, "content", "") or ""): + to_delete.append(between) + return to_delete + + +def setup_history_routes(session_manager) -> APIRouter: + router = APIRouter(tags=["history"]) + + def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]: + entry = {"role": m.role, "content": _history_display_content(m.content)} + meta = {} + if m.meta_data: + try: + meta = json.loads(m.meta_data) or {} + except (json.JSONDecodeError, ValueError): + meta = {} + if m.timestamp and "timestamp" not in meta: + meta["timestamp"] = m.timestamp.isoformat() + "Z" + if meta: + entry["metadata"] = meta + return entry + + @router.get("/api/history/{session_id}") + async def get_session_history( + request: Request, + session_id: str, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Dict[str, Any]: + _verify_session_owner(request, session_id) + if limit is not None: + page_limit = max(1, min(int(limit), 100)) + db = SessionLocal() + try: + db_session = db.query(DbSession).filter(DbSession.id == session_id).first() + if db_session is None: + raise HTTPException(404, f"Session '{session_id}' not found") + + total = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .count() + ) + page_offset = int(offset) if offset is not None else max(total - page_limit, 0) + page_offset = max(0, min(page_offset, total)) + rows = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .order_by(DbChatMessage.timestamp) + .offset(page_offset) + .limit(page_limit) + .all() + ) + history_dict = [ + entry for entry in (_db_history_entry(m) for m in rows) + if not (entry.get("metadata") or {}).get("hidden") + ] + return { + "history": history_dict, + "model": db_session.model, + "endpoint_url": db_session.endpoint_url, + "name": db_session.name, + "offset": page_offset, + "limit": page_limit, + "total": total, + "has_more_before": page_offset > 0, + "has_more_after": page_offset + len(rows) < total, + } + finally: + db.close() + + try: + session = session_manager.get_session(session_id) + except KeyError: + raise HTTPException(404, f"Session '{session_id}' not found") + + history_dict = [] + for msg in session.history: + if isinstance(msg, ChatMessage): + # Skip hidden messages (e.g. compaction summaries for AI context) + if msg.metadata and msg.metadata.get("hidden"): + continue + entry = {"role": msg.role, "content": _history_display_content(msg.content)} + if msg.metadata: + entry["metadata"] = msg.metadata + history_dict.append(entry) + elif isinstance(msg, dict): + if msg.get("metadata", {}).get("hidden"): + continue + entry = { + "role": msg.get("role", ""), + "content": _history_display_content(msg.get("content", "")), + } + if msg.get("metadata"): + entry["metadata"] = msg["metadata"] + history_dict.append(entry) + + # Fallback: load from DB if in-memory is empty + if not history_dict: + db = SessionLocal() + try: + db_messages = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .order_by(DbChatMessage.timestamp) + .all() + ) + db_history = [] + for m in db_messages: + db_history.append(_db_history_entry(m)) + if db_history: + # Rebuild in-memory history from the full set so hidden + # messages (e.g. compaction summaries) are kept for AI context. + session.history = [ + ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata")) + for m in db_history + ] + # Response excludes hidden messages, matching the in-memory path. + history_dict = [ + m for m in db_history + if not (m.get("metadata") or {}).get("hidden") + ] + except Exception as e: + logger.error(f"DB fallback failed for {session_id}: {e}") + finally: + db.close() + + return { + "history": history_dict, + "model": session.model, + "endpoint_url": session.endpoint_url, + "name": session.name, + } + + @router.post("/api/session/{session_id}/truncate") + async def truncate_session(request: Request, session_id: str): + _verify_session_owner(request, session_id) + try: + body = await request.json() + keep_count = body.get("keep_count", 0) + result = session_manager.truncate_messages(session_id, keep_count) + return {"status": "ok", "kept": keep_count, "truncated": result} + except KeyError: + raise HTTPException(404, "Session not found") + except Exception as e: + logger.error(f"Truncate error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.post("/api/session/{session_id}/message") + async def add_message(request: Request, session_id: str): + """Add a message to a session (for slash command persistence).""" + _verify_session_owner(request, session_id) + try: + body = await request.json() + role = body.get("role", "assistant") + content = body.get("content", "") + if not content: + raise HTTPException(400, "content is required") + msg = ChatMessage(role=role, content=content, metadata=body.get("metadata")) + session_manager.add_message(session_id, msg) + return {"status": "ok"} + except KeyError: + raise HTTPException(404, "Session not found") + + @router.post("/api/session/{session_id}/delete-messages") + async def delete_messages(request: Request, session_id: str): + """Delete specific messages by DB ID (or legacy index).""" + _verify_session_owner(request, session_id) + try: + body = await request.json() + msg_ids = body.get("msg_ids", []) + indices = body.get("indices") # legacy fallback + + session = session_manager.get_session(session_id) + db = SessionLocal() + try: + if msg_ids: + # New ID-based delete + deleted = 0 + for mid in msg_ids: + db_msg = db.query(DbChatMessage).filter( + DbChatMessage.id == mid, + DbChatMessage.session_id == session_id, + ).first() + if db_msg: + db.delete(db_msg) + deleted += 1 + + # Remove from in-memory history by matching _db_id + def _get_db_id(m): + meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None) + return meta.get('_db_id') if isinstance(meta, dict) else None + session.history = [m for m in session.history if _get_db_id(m) not in msg_ids] + elif indices: + # Legacy index-based delete + indices = sorted(indices, reverse=True) + db_messages = db.query(DbChatMessage).filter( + DbChatMessage.session_id == session_id + ).order_by(DbChatMessage.timestamp).all() + + deleted = 0 + for idx in indices: + if 0 <= idx < len(db_messages): + db.delete(db_messages[idx]) + deleted += 1 + if 0 <= idx < len(session.history): + session.history.pop(idx) + else: + return {"status": "ok", "deleted": 0} + + session.message_count = len(session.history) + db_session = db.query(DbSession).filter(DbSession.id == session_id).first() + if db_session: + db_session.message_count = len(session.history) + from datetime import datetime, timezone + db_session.updated_at = datetime.now(timezone.utc) + + db.commit() + return {"status": "ok", "deleted": deleted} + finally: + db.close() + except KeyError: + raise HTTPException(404, "Session not found") + except Exception as e: + logger.error(f"Delete messages error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.post("/api/session/{session_id}/edit-message") + async def edit_message(request: Request, session_id: str): + """Edit the content of a message by its database ID.""" + _verify_session_owner(request, session_id) + try: + body = await request.json() + msg_id = body.get("msg_id") + content = body.get("content") + if not msg_id or content is None: + raise HTTPException(400, "msg_id and content are required") + + session = session_manager.get_session(session_id) + db = SessionLocal() + try: + db_msg = db.query(DbChatMessage).filter( + DbChatMessage.id == msg_id, + DbChatMessage.session_id == session_id, + ).first() + if not db_msg: + raise HTTPException(404, "Message not found") + + db_msg.content = content + meta = {} + if db_msg.meta_data: + try: meta = json.loads(db_msg.meta_data) + except (json.JSONDecodeError, ValueError): pass + meta['edited'] = True + db_msg.meta_data = json.dumps(meta) + + # Update in-memory history by matching _db_id + for hmsg in session.history: + hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata') + if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id: + if isinstance(hmsg, ChatMessage): + hmsg.content = content + hmsg.metadata['edited'] = True + elif isinstance(hmsg, dict): + hmsg['content'] = content + hmsg['metadata']['edited'] = True + break + + db.commit() + return {"status": "ok"} + finally: + db.close() + except KeyError: + raise HTTPException(404, "Session not found") + except HTTPException: + raise + except Exception as e: + logger.error(f"Edit message error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.post("/api/session/{session_id}/mark-stopped") + async def mark_stopped(request: Request, session_id: str): + """Mark the last assistant message as stopped by user.""" + _verify_session_owner(request, session_id) + try: + session = session_manager.get_session(session_id) + # Find last assistant message and add stopped metadata + for msg in reversed(session.history): + if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \ + (isinstance(msg, dict) and msg.get('role') == 'assistant'): + if isinstance(msg, ChatMessage): + if not msg.metadata: + msg.metadata = {} + msg.metadata['stopped'] = True + if not msg.metadata.get('model'): + msg.metadata['model'] = session.model + else: + if 'metadata' not in msg: + msg['metadata'] = {} + msg['metadata']['stopped'] = True + if not msg['metadata'].get('model'): + msg['metadata']['model'] = session.model + break + # Also update in DB + db = SessionLocal() + try: + import json as _json + db_messages = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant') + .order_by(DbChatMessage.timestamp.desc()) + .first() + ) + if db_messages: + meta = {} + if db_messages.meta_data: + try: + meta = _json.loads(db_messages.meta_data) + except (json.JSONDecodeError, ValueError): + pass + meta['stopped'] = True + if not meta.get('model'): + meta['model'] = session.model + db_messages.meta_data = _json.dumps(meta) + db.commit() + finally: + db.close() + session_manager.save_sessions() + return {"status": "ok"} + except KeyError: + raise HTTPException(404, "Session not found") + except Exception as e: + logger.error(f"Mark stopped error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.post("/api/session/{session_id}/update-last-meta") + async def update_last_meta(request: Request, session_id: str): + """Merge metadata into the last assistant message (e.g. save variants).""" + _verify_session_owner(request, session_id) + try: + body = await request.json() + meta_update = body.get("metadata", {}) + session = session_manager.get_session(session_id) + + # Update in-memory + for msg in reversed(session.history): + if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \ + (isinstance(msg, dict) and msg.get('role') == 'assistant'): + if isinstance(msg, ChatMessage): + if not msg.metadata: + msg.metadata = {} + msg.metadata.update(meta_update) + else: + if 'metadata' not in msg: + msg['metadata'] = {} + msg['metadata'].update(meta_update) + break + + # Update in DB + db = SessionLocal() + try: + import json as _json + db_msg = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant') + .order_by(DbChatMessage.timestamp.desc()) + .first() + ) + if db_msg: + meta = {} + if db_msg.meta_data: + try: meta = _json.loads(db_msg.meta_data) + except (json.JSONDecodeError, ValueError): pass + meta.update(meta_update) + db_msg.meta_data = _json.dumps(meta) + db.commit() + finally: + db.close() + session_manager.save_sessions() + return {"status": "ok"} + except KeyError: + raise HTTPException(404, "Session not found") + except Exception as e: + logger.error(f"Update last meta error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.post("/api/session/{session_id}/merge-last-assistant") + async def merge_last_assistant(request: Request, session_id: str): + """Merge the last two assistant messages into one (for continue).""" + _verify_session_owner(request, session_id) + try: + body = await request.json() + separator = body.get("separator", "\n\n") + session = session_manager.get_session(session_id) + + # Find last two assistant messages in-memory + ai_indices = [] + for i, msg in enumerate(session.history): + role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '') + if role == 'assistant': + ai_indices.append(i) + + if len(ai_indices) < 2: + return {"status": "ok", "merged": False} + + idx1, idx2 = ai_indices[-2], ai_indices[-1] + msg1, msg2 = session.history[idx1], session.history[idx2] + + content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '') + content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '') + merged_content = content1 + separator + content2 + + # Merge metadata + meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {} + meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {} + merged_meta = {**meta1, **meta2} + merged_meta.pop('stopped', None) # no longer stopped after continue + + # Update first message, remove second + if isinstance(msg1, ChatMessage): + msg1.content = merged_content + msg1.metadata = merged_meta + else: + msg1['content'] = merged_content + msg1['metadata'] = merged_meta + + # Also remove the hidden "continue" user message between them if present + # It's the message at idx2-1 if it's a user message with continue text + remove_indices = [idx2] + if idx2 - 1 > idx1: + between = session.history[idx2 - 1] + between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '') + between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '') + if between_role == 'user' and 'previous response was interrupted' in between_content: + remove_indices.insert(0, idx2 - 1) + + for ri in sorted(remove_indices, reverse=True): + session.history.pop(ri) + + # Update DB + db = SessionLocal() + try: + import json as _json + db_messages = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .order_by(DbChatMessage.timestamp) + .all() + ) + # Find last two assistant messages in DB + ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant'] + if len(ai_db) >= 2: + (_, db1), (_, db2) = ai_db[-2], ai_db[-1] + db1.content = merged_content + db1.meta_data = _json.dumps(merged_meta) + + # Mirror the in-memory deletion: remove the second assistant + # message and ONLY the "continue" user message between them + # (not arbitrary tool/system/user rows). The old + # range-delete destroyed every row between the two assistant + # messages, desyncing the DB from the in-memory history. + for _row in _merge_continue_rows_to_delete(db_messages, db1, db2): + db.delete(_row) + + db.commit() + finally: + db.close() + session_manager.save_sessions() + return {"status": "ok", "merged": True} + except KeyError: + raise HTTPException(404, "Session not found") + except Exception as e: + logger.error(f"Merge assistant error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.post("/api/session/{session_id}/fork") + async def fork_session(request: Request, session_id: str): + """Create a new session with messages copied up to keep_count.""" + _verify_session_owner(request, session_id) + try: + body = await request.json() + keep_count = body.get("keep_count", 0) + + # Get the source session + source = session_manager.sessions.get(session_id) + if not source: + raise HTTPException(404, "Session not found") + + # Create new session + new_id = str(uuid.uuid4()) + fork_name = f"\u2ADD {source.name}" + new_session = session_manager.create_session( + session_id=new_id, + name=fork_name, + endpoint_url=source.endpoint_url, + model=source.model, + rag=False, + owner=getattr(source, 'owner', None), + ) + + # Copy messages up to keep_count + msgs_to_copy = source.history[:keep_count] + for msg in msgs_to_copy: + # Copy the metadata dict. Sharing it would let the fork's + # persistence (add_message -> _persist_message stamps + # _db_id/timestamp onto the dict) mutate the SOURCE session's + # in-memory messages, corrupting their _db_id and breaking + # edit/delete-by-id on the original conversation. + meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None + new_session.add_message(ChatMessage(msg.role, msg.content, meta)) + try: + from src.event_bus import fire_event + fire_event("session_created", getattr(source, 'owner', None)) + except Exception: + logger.debug("session_created event dispatch failed", exc_info=True) + + return { + "status": "ok", + "id": new_id, + "name": fork_name, + "kept": len(msgs_to_copy), + } + except HTTPException: + raise + except Exception as e: + logger.error(f"Fork error {session_id}: {e}") + raise HTTPException(500, str(e)) + + @router.get("/api/conversations/topics") + async def get_conversation_topics(request: Request) -> Dict[str, Any]: + from src.auth_helpers import require_user + user = require_user(request) + try: + return analyze_topics(session_manager, owner=user or None) + except Exception as e: + raise HTTPException(500, f"Topic analysis failed: {e}") + + @router.post("/api/session/{session_id}/compact") + async def compact_session(request: Request, session_id: str): + """Manually trigger context compaction for a session.""" + _verify_session_owner(request, session_id) + from src.auth_helpers import effective_user + owner = effective_user(request) + try: + session = session_manager.get_session(session_id) + except KeyError: + raise HTTPException(404, "Session not found") + _reject_compact_during_active_run(session_id) + + try: + from src.model_context import estimate_tokens, get_context_length + from src.llm_core import llm_call_async + from src.endpoint_resolver import resolve_endpoint + + if len(session.history) < 6: + return {"status": "ok", "message": "Not enough messages to compact"} + + ctx_len = get_context_length(session.endpoint_url, session.model) + messages_before = session.get_context_messages() + used_before = estimate_tokens(messages_before) + pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0 + msg_count_before = len(session.history) + + # Keep only last 4 messages, summarize the rest + keep_count = 4 + older = session.history[:-keep_count] + recent = session.history[-keep_count:] + + # Build text to summarize + convo_text = "\n".join( + f"{_message_role(m).upper()}: " + f"{_message_text(m)[:2000]}" + for m in older + ) + + # Use utility model if available + util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None) + compact_url = util_url or session.endpoint_url + compact_model = util_model or session.model + compact_headers = util_headers if util_url else session.headers + + from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT + compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or "")) + sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1)) + summary = await llm_call_async( + compact_url, compact_model, + [ + {"role": "system", "content": sys_prompt}, + {"role": "user", "content": convo_text}, + ], + temperature=0.2, max_tokens=1024, + headers=compact_headers, timeout=30, + ) + + # Replace session history: summary as system message + recent messages + # System message holds the full summary for AI context + system_summary = ChatMessage( + role="system", + content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}", + metadata={"compacted": True, "hidden": True}, + ) + # Visible assistant message just shows stats + summary_msg = ChatMessage( + role="assistant", + content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.", + metadata={"compacted": True, "messages_removed": len(older)}, + ) + new_history = [system_summary, summary_msg] + list(recent) + session.history = new_history + session.message_count = len(session.history) + logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})") + + # Update DB: delete old messages, insert summary + db = SessionLocal() + try: + db_msgs = db.query(DbChatMessage).filter( + DbChatMessage.session_id == session_id + ).order_by(DbChatMessage.timestamp).all() + + # Delete all but the last keep_count + for m in db_msgs[:-keep_count]: + db.delete(m) + + # Insert system summary (hidden, for AI context) and visible summary + import json as _json + import uuid + from datetime import datetime, timezone + now = datetime.now(timezone.utc) + db_sys_summary = DbChatMessage( + id=str(uuid.uuid4()), + session_id=session_id, + role="system", + content=system_summary.content, + meta_data=_json.dumps(system_summary.metadata), + timestamp=now, + ) + db.add(db_sys_summary) + db_summary = DbChatMessage( + id=str(uuid.uuid4()), + session_id=session_id, + role="assistant", + content=summary_msg.content, + meta_data=_json.dumps(summary_msg.metadata), + timestamp=now, + ) + db.add(db_summary) + + # Update session record + db_session = db.query(DbSession).filter(DbSession.id == session_id).first() + if db_session: + db_session.message_count = len(session.history) + db_session.updated_at = datetime.now(timezone.utc) + db.commit() + finally: + db.close() + + session_manager.save_sessions() + + used_after = estimate_tokens(session.get_context_messages()) + pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0 + + return { + "status": "ok", + "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)", + "before": pct_before, + "after": pct_after, + } + + except Exception as e: + logger.error(f"Manual compact error {session_id}: {e}") + raise HTTPException(500, str(e)) + + return router diff --git a/routes/history_routes.py b/routes/history_routes.py index 2324ae286..3362e70de 100644 --- a/routes/history_routes.py +++ b/routes/history_routes.py @@ -1,768 +1,17 @@ -"""History routes — session history, truncation, fork, conversation topics.""" +"""Backward-compat shim — canonical location is routes/history/history_routes.py. -import json -import uuid -import logging -import re -from typing import Dict, Any, Optional +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.history_routes``, ``from routes.history_routes import X``, +``importlib.import_module("routes.history_routes")``, and the +``import ... as history_routes`` + ``monkeypatch.setattr(history_routes, ...)`` +pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py +all operate on the *same* object the application actually uses. Keeps existing +import paths working after slice 2d (#4082/#4071). Source-introspection tests +read the canonical file by path. +""" -from fastapi import APIRouter, Request, HTTPException +import sys as _sys -from core.models import ChatMessage -from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession -from src.topic_analyzer import analyze_topics -from routes.session_routes import ( - _message_role, - _message_text, - _reject_compact_during_active_run, - _verify_session_owner, -) +from routes.history import history_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - -_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000 -_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+") - - -def _history_display_content(content: Any) -> Any: - """Return a lightweight browser-display copy of stored message content. - - Older multimodal user messages may be persisted as a JSON *string* - containing image_url blocks with inline base64 image bytes. Those bytes are - needed for model calls when the turn is first sent, but they should not be - sent back through /api/history every time the user opens the chat. The - attachment metadata already carries file ids/names for the UI cards. - """ - if isinstance(content, list): - text_parts = [] - omitted_media = 0 - for block in content: - if not isinstance(block, dict): - continue - if block.get("type") == "text": - text = block.get("text") - if isinstance(text, str) and text: - text_parts.append(text) - elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}: - omitted_media += 1 - text = "\n".join(text_parts).strip() - if omitted_media and not text: - return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]" - return text - - if not isinstance(content, str): - return content - if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content: - return content - - stripped = content.lstrip() - if stripped.startswith("["): - try: - blocks = json.loads(content) - except (json.JSONDecodeError, TypeError, ValueError): - blocks = None - if isinstance(blocks, list): - text_parts = [] - for block in blocks: - if not isinstance(block, dict): - continue - if block.get("type") == "text": - text = block.get("text") - if isinstance(text, str) and text: - text_parts.append(text) - if text_parts: - return "\n".join(text_parts).strip() - - if "data:image/" in content: - return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content) - return content - - -def _merge_continue_rows_to_delete(db_messages, db1, db2): - """DB rows to delete when merging the last two assistant messages. - - Always the second assistant message (db2), plus ONLY the single - intervening "continue" user message (the one carrying "previous response - was interrupted") — matching the in-memory merge. The previous code - deleted the whole index range between the two assistant rows, destroying - any tool/system/user messages in between and desyncing the DB from the - in-memory history. - """ - to_delete = [db2] - i1 = next((i for i, m in enumerate(db_messages) if m is db1), None) - i2 = next((i for i, m in enumerate(db_messages) if m is db2), None) - if i1 is not None and i2 is not None and i2 - 1 > i1: - between = db_messages[i2 - 1] - if getattr(between, "role", "") == "user" and "previous response was interrupted" in (getattr(between, "content", "") or ""): - to_delete.append(between) - return to_delete - - -def setup_history_routes(session_manager) -> APIRouter: - router = APIRouter(tags=["history"]) - - def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]: - entry = {"role": m.role, "content": _history_display_content(m.content)} - meta = {} - if m.meta_data: - try: - meta = json.loads(m.meta_data) or {} - except (json.JSONDecodeError, ValueError): - meta = {} - if m.timestamp and "timestamp" not in meta: - meta["timestamp"] = m.timestamp.isoformat() + "Z" - if meta: - entry["metadata"] = meta - return entry - - @router.get("/api/history/{session_id}") - async def get_session_history( - request: Request, - session_id: str, - limit: Optional[int] = None, - offset: Optional[int] = None, - ) -> Dict[str, Any]: - _verify_session_owner(request, session_id) - if limit is not None: - page_limit = max(1, min(int(limit), 100)) - db = SessionLocal() - try: - db_session = db.query(DbSession).filter(DbSession.id == session_id).first() - if db_session is None: - raise HTTPException(404, f"Session '{session_id}' not found") - - total = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id) - .count() - ) - page_offset = int(offset) if offset is not None else max(total - page_limit, 0) - page_offset = max(0, min(page_offset, total)) - rows = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id) - .order_by(DbChatMessage.timestamp) - .offset(page_offset) - .limit(page_limit) - .all() - ) - history_dict = [ - entry for entry in (_db_history_entry(m) for m in rows) - if not (entry.get("metadata") or {}).get("hidden") - ] - return { - "history": history_dict, - "model": db_session.model, - "endpoint_url": db_session.endpoint_url, - "name": db_session.name, - "offset": page_offset, - "limit": page_limit, - "total": total, - "has_more_before": page_offset > 0, - "has_more_after": page_offset + len(rows) < total, - } - finally: - db.close() - - try: - session = session_manager.get_session(session_id) - except KeyError: - raise HTTPException(404, f"Session '{session_id}' not found") - - history_dict = [] - for msg in session.history: - if isinstance(msg, ChatMessage): - # Skip hidden messages (e.g. compaction summaries for AI context) - if msg.metadata and msg.metadata.get("hidden"): - continue - entry = {"role": msg.role, "content": _history_display_content(msg.content)} - if msg.metadata: - entry["metadata"] = msg.metadata - history_dict.append(entry) - elif isinstance(msg, dict): - if msg.get("metadata", {}).get("hidden"): - continue - entry = { - "role": msg.get("role", ""), - "content": _history_display_content(msg.get("content", "")), - } - if msg.get("metadata"): - entry["metadata"] = msg["metadata"] - history_dict.append(entry) - - # Fallback: load from DB if in-memory is empty - if not history_dict: - db = SessionLocal() - try: - db_messages = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id) - .order_by(DbChatMessage.timestamp) - .all() - ) - db_history = [] - for m in db_messages: - db_history.append(_db_history_entry(m)) - if db_history: - # Rebuild in-memory history from the full set so hidden - # messages (e.g. compaction summaries) are kept for AI context. - session.history = [ - ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata")) - for m in db_history - ] - # Response excludes hidden messages, matching the in-memory path. - history_dict = [ - m for m in db_history - if not (m.get("metadata") or {}).get("hidden") - ] - except Exception as e: - logger.error(f"DB fallback failed for {session_id}: {e}") - finally: - db.close() - - return { - "history": history_dict, - "model": session.model, - "endpoint_url": session.endpoint_url, - "name": session.name, - } - - @router.post("/api/session/{session_id}/truncate") - async def truncate_session(request: Request, session_id: str): - _verify_session_owner(request, session_id) - try: - body = await request.json() - keep_count = body.get("keep_count", 0) - result = session_manager.truncate_messages(session_id, keep_count) - return {"status": "ok", "kept": keep_count, "truncated": result} - except KeyError: - raise HTTPException(404, "Session not found") - except Exception as e: - logger.error(f"Truncate error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.post("/api/session/{session_id}/message") - async def add_message(request: Request, session_id: str): - """Add a message to a session (for slash command persistence).""" - _verify_session_owner(request, session_id) - try: - body = await request.json() - role = body.get("role", "assistant") - content = body.get("content", "") - if not content: - raise HTTPException(400, "content is required") - msg = ChatMessage(role=role, content=content, metadata=body.get("metadata")) - session_manager.add_message(session_id, msg) - return {"status": "ok"} - except KeyError: - raise HTTPException(404, "Session not found") - - @router.post("/api/session/{session_id}/delete-messages") - async def delete_messages(request: Request, session_id: str): - """Delete specific messages by DB ID (or legacy index).""" - _verify_session_owner(request, session_id) - try: - body = await request.json() - msg_ids = body.get("msg_ids", []) - indices = body.get("indices") # legacy fallback - - session = session_manager.get_session(session_id) - db = SessionLocal() - try: - if msg_ids: - # New ID-based delete - deleted = 0 - for mid in msg_ids: - db_msg = db.query(DbChatMessage).filter( - DbChatMessage.id == mid, - DbChatMessage.session_id == session_id, - ).first() - if db_msg: - db.delete(db_msg) - deleted += 1 - - # Remove from in-memory history by matching _db_id - def _get_db_id(m): - meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None) - return meta.get('_db_id') if isinstance(meta, dict) else None - session.history = [m for m in session.history if _get_db_id(m) not in msg_ids] - elif indices: - # Legacy index-based delete - indices = sorted(indices, reverse=True) - db_messages = db.query(DbChatMessage).filter( - DbChatMessage.session_id == session_id - ).order_by(DbChatMessage.timestamp).all() - - deleted = 0 - for idx in indices: - if 0 <= idx < len(db_messages): - db.delete(db_messages[idx]) - deleted += 1 - if 0 <= idx < len(session.history): - session.history.pop(idx) - else: - return {"status": "ok", "deleted": 0} - - session.message_count = len(session.history) - db_session = db.query(DbSession).filter(DbSession.id == session_id).first() - if db_session: - db_session.message_count = len(session.history) - from datetime import datetime, timezone - db_session.updated_at = datetime.now(timezone.utc) - - db.commit() - return {"status": "ok", "deleted": deleted} - finally: - db.close() - except KeyError: - raise HTTPException(404, "Session not found") - except Exception as e: - logger.error(f"Delete messages error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.post("/api/session/{session_id}/edit-message") - async def edit_message(request: Request, session_id: str): - """Edit the content of a message by its database ID.""" - _verify_session_owner(request, session_id) - try: - body = await request.json() - msg_id = body.get("msg_id") - content = body.get("content") - if not msg_id or content is None: - raise HTTPException(400, "msg_id and content are required") - - session = session_manager.get_session(session_id) - db = SessionLocal() - try: - db_msg = db.query(DbChatMessage).filter( - DbChatMessage.id == msg_id, - DbChatMessage.session_id == session_id, - ).first() - if not db_msg: - raise HTTPException(404, "Message not found") - - db_msg.content = content - meta = {} - if db_msg.meta_data: - try: meta = json.loads(db_msg.meta_data) - except (json.JSONDecodeError, ValueError): pass - meta['edited'] = True - db_msg.meta_data = json.dumps(meta) - - # Update in-memory history by matching _db_id - for hmsg in session.history: - hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata') - if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id: - if isinstance(hmsg, ChatMessage): - hmsg.content = content - hmsg.metadata['edited'] = True - elif isinstance(hmsg, dict): - hmsg['content'] = content - hmsg['metadata']['edited'] = True - break - - db.commit() - return {"status": "ok"} - finally: - db.close() - except KeyError: - raise HTTPException(404, "Session not found") - except HTTPException: - raise - except Exception as e: - logger.error(f"Edit message error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.post("/api/session/{session_id}/mark-stopped") - async def mark_stopped(request: Request, session_id: str): - """Mark the last assistant message as stopped by user.""" - _verify_session_owner(request, session_id) - try: - session = session_manager.get_session(session_id) - # Find last assistant message and add stopped metadata - for msg in reversed(session.history): - if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \ - (isinstance(msg, dict) and msg.get('role') == 'assistant'): - if isinstance(msg, ChatMessage): - if not msg.metadata: - msg.metadata = {} - msg.metadata['stopped'] = True - if not msg.metadata.get('model'): - msg.metadata['model'] = session.model - else: - if 'metadata' not in msg: - msg['metadata'] = {} - msg['metadata']['stopped'] = True - if not msg['metadata'].get('model'): - msg['metadata']['model'] = session.model - break - # Also update in DB - db = SessionLocal() - try: - import json as _json - db_messages = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant') - .order_by(DbChatMessage.timestamp.desc()) - .first() - ) - if db_messages: - meta = {} - if db_messages.meta_data: - try: - meta = _json.loads(db_messages.meta_data) - except (json.JSONDecodeError, ValueError): - pass - meta['stopped'] = True - if not meta.get('model'): - meta['model'] = session.model - db_messages.meta_data = _json.dumps(meta) - db.commit() - finally: - db.close() - session_manager.save_sessions() - return {"status": "ok"} - except KeyError: - raise HTTPException(404, "Session not found") - except Exception as e: - logger.error(f"Mark stopped error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.post("/api/session/{session_id}/update-last-meta") - async def update_last_meta(request: Request, session_id: str): - """Merge metadata into the last assistant message (e.g. save variants).""" - _verify_session_owner(request, session_id) - try: - body = await request.json() - meta_update = body.get("metadata", {}) - session = session_manager.get_session(session_id) - - # Update in-memory - for msg in reversed(session.history): - if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \ - (isinstance(msg, dict) and msg.get('role') == 'assistant'): - if isinstance(msg, ChatMessage): - if not msg.metadata: - msg.metadata = {} - msg.metadata.update(meta_update) - else: - if 'metadata' not in msg: - msg['metadata'] = {} - msg['metadata'].update(meta_update) - break - - # Update in DB - db = SessionLocal() - try: - import json as _json - db_msg = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant') - .order_by(DbChatMessage.timestamp.desc()) - .first() - ) - if db_msg: - meta = {} - if db_msg.meta_data: - try: meta = _json.loads(db_msg.meta_data) - except (json.JSONDecodeError, ValueError): pass - meta.update(meta_update) - db_msg.meta_data = _json.dumps(meta) - db.commit() - finally: - db.close() - session_manager.save_sessions() - return {"status": "ok"} - except KeyError: - raise HTTPException(404, "Session not found") - except Exception as e: - logger.error(f"Update last meta error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.post("/api/session/{session_id}/merge-last-assistant") - async def merge_last_assistant(request: Request, session_id: str): - """Merge the last two assistant messages into one (for continue).""" - _verify_session_owner(request, session_id) - try: - body = await request.json() - separator = body.get("separator", "\n\n") - session = session_manager.get_session(session_id) - - # Find last two assistant messages in-memory - ai_indices = [] - for i, msg in enumerate(session.history): - role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '') - if role == 'assistant': - ai_indices.append(i) - - if len(ai_indices) < 2: - return {"status": "ok", "merged": False} - - idx1, idx2 = ai_indices[-2], ai_indices[-1] - msg1, msg2 = session.history[idx1], session.history[idx2] - - content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '') - content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '') - merged_content = content1 + separator + content2 - - # Merge metadata - meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {} - meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {} - merged_meta = {**meta1, **meta2} - merged_meta.pop('stopped', None) # no longer stopped after continue - - # Update first message, remove second - if isinstance(msg1, ChatMessage): - msg1.content = merged_content - msg1.metadata = merged_meta - else: - msg1['content'] = merged_content - msg1['metadata'] = merged_meta - - # Also remove the hidden "continue" user message between them if present - # It's the message at idx2-1 if it's a user message with continue text - remove_indices = [idx2] - if idx2 - 1 > idx1: - between = session.history[idx2 - 1] - between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '') - between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '') - if between_role == 'user' and 'previous response was interrupted' in between_content: - remove_indices.insert(0, idx2 - 1) - - for ri in sorted(remove_indices, reverse=True): - session.history.pop(ri) - - # Update DB - db = SessionLocal() - try: - import json as _json - db_messages = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id) - .order_by(DbChatMessage.timestamp) - .all() - ) - # Find last two assistant messages in DB - ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant'] - if len(ai_db) >= 2: - (_, db1), (_, db2) = ai_db[-2], ai_db[-1] - db1.content = merged_content - db1.meta_data = _json.dumps(merged_meta) - - # Mirror the in-memory deletion: remove the second assistant - # message and ONLY the "continue" user message between them - # (not arbitrary tool/system/user rows). The old - # range-delete destroyed every row between the two assistant - # messages, desyncing the DB from the in-memory history. - for _row in _merge_continue_rows_to_delete(db_messages, db1, db2): - db.delete(_row) - - db.commit() - finally: - db.close() - session_manager.save_sessions() - return {"status": "ok", "merged": True} - except KeyError: - raise HTTPException(404, "Session not found") - except Exception as e: - logger.error(f"Merge assistant error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.post("/api/session/{session_id}/fork") - async def fork_session(request: Request, session_id: str): - """Create a new session with messages copied up to keep_count.""" - _verify_session_owner(request, session_id) - try: - body = await request.json() - keep_count = body.get("keep_count", 0) - - # Get the source session - source = session_manager.sessions.get(session_id) - if not source: - raise HTTPException(404, "Session not found") - - # Create new session - new_id = str(uuid.uuid4()) - fork_name = f"\u2ADD {source.name}" - new_session = session_manager.create_session( - session_id=new_id, - name=fork_name, - endpoint_url=source.endpoint_url, - model=source.model, - rag=False, - owner=getattr(source, 'owner', None), - ) - - # Copy messages up to keep_count - msgs_to_copy = source.history[:keep_count] - for msg in msgs_to_copy: - # Copy the metadata dict. Sharing it would let the fork's - # persistence (add_message -> _persist_message stamps - # _db_id/timestamp onto the dict) mutate the SOURCE session's - # in-memory messages, corrupting their _db_id and breaking - # edit/delete-by-id on the original conversation. - meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None - new_session.add_message(ChatMessage(msg.role, msg.content, meta)) - try: - from src.event_bus import fire_event - fire_event("session_created", getattr(source, 'owner', None)) - except Exception: - logger.debug("session_created event dispatch failed", exc_info=True) - - return { - "status": "ok", - "id": new_id, - "name": fork_name, - "kept": len(msgs_to_copy), - } - except HTTPException: - raise - except Exception as e: - logger.error(f"Fork error {session_id}: {e}") - raise HTTPException(500, str(e)) - - @router.get("/api/conversations/topics") - async def get_conversation_topics(request: Request) -> Dict[str, Any]: - from src.auth_helpers import require_user - user = require_user(request) - try: - return analyze_topics(session_manager, owner=user or None) - except Exception as e: - raise HTTPException(500, f"Topic analysis failed: {e}") - - @router.post("/api/session/{session_id}/compact") - async def compact_session(request: Request, session_id: str): - """Manually trigger context compaction for a session.""" - _verify_session_owner(request, session_id) - from src.auth_helpers import effective_user - owner = effective_user(request) - try: - session = session_manager.get_session(session_id) - except KeyError: - raise HTTPException(404, "Session not found") - _reject_compact_during_active_run(session_id) - - try: - from src.model_context import estimate_tokens, get_context_length - from src.llm_core import llm_call_async - from src.endpoint_resolver import resolve_endpoint - - if len(session.history) < 6: - return {"status": "ok", "message": "Not enough messages to compact"} - - ctx_len = get_context_length(session.endpoint_url, session.model) - messages_before = session.get_context_messages() - used_before = estimate_tokens(messages_before) - pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0 - msg_count_before = len(session.history) - - # Keep only last 4 messages, summarize the rest - keep_count = 4 - older = session.history[:-keep_count] - recent = session.history[-keep_count:] - - # Build text to summarize - convo_text = "\n".join( - f"{_message_role(m).upper()}: " - f"{_message_text(m)[:2000]}" - for m in older - ) - - # Use utility model if available - util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None) - compact_url = util_url or session.endpoint_url - compact_model = util_model or session.model - compact_headers = util_headers if util_url else session.headers - - from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT - compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or "")) - sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1)) - summary = await llm_call_async( - compact_url, compact_model, - [ - {"role": "system", "content": sys_prompt}, - {"role": "user", "content": convo_text}, - ], - temperature=0.2, max_tokens=1024, - headers=compact_headers, timeout=30, - ) - - # Replace session history: summary as system message + recent messages - # System message holds the full summary for AI context - system_summary = ChatMessage( - role="system", - content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}", - metadata={"compacted": True, "hidden": True}, - ) - # Visible assistant message just shows stats - summary_msg = ChatMessage( - role="assistant", - content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.", - metadata={"compacted": True, "messages_removed": len(older)}, - ) - new_history = [system_summary, summary_msg] + list(recent) - session.history = new_history - session.message_count = len(session.history) - logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})") - - # Update DB: delete old messages, insert summary - db = SessionLocal() - try: - db_msgs = db.query(DbChatMessage).filter( - DbChatMessage.session_id == session_id - ).order_by(DbChatMessage.timestamp).all() - - # Delete all but the last keep_count - for m in db_msgs[:-keep_count]: - db.delete(m) - - # Insert system summary (hidden, for AI context) and visible summary - import json as _json - import uuid - from datetime import datetime, timezone - now = datetime.now(timezone.utc) - db_sys_summary = DbChatMessage( - id=str(uuid.uuid4()), - session_id=session_id, - role="system", - content=system_summary.content, - meta_data=_json.dumps(system_summary.metadata), - timestamp=now, - ) - db.add(db_sys_summary) - db_summary = DbChatMessage( - id=str(uuid.uuid4()), - session_id=session_id, - role="assistant", - content=summary_msg.content, - meta_data=_json.dumps(summary_msg.metadata), - timestamp=now, - ) - db.add(db_summary) - - # Update session record - db_session = db.query(DbSession).filter(DbSession.id == session_id).first() - if db_session: - db_session.message_count = len(session.history) - db_session.updated_at = datetime.now(timezone.utc) - db.commit() - finally: - db.close() - - session_manager.save_sessions() - - used_after = estimate_tokens(session.get_context_messages()) - pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0 - - return { - "status": "ok", - "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)", - "before": pct_before, - "after": pct_after, - } - - except Exception as e: - logger.error(f"Manual compact error {session_id}: {e}") - raise HTTPException(500, str(e)) - - return router +_sys.modules[__name__] = _canonical diff --git a/tests/test_history_db_fallback_hidden.py b/tests/test_history_db_fallback_hidden.py index 7e43d16ae..855d68c74 100644 --- a/tests/test_history_db_fallback_hidden.py +++ b/tests/test_history_db_fallback_hidden.py @@ -16,7 +16,7 @@ in this repo do). import ast from pathlib import Path -SRC = Path(__file__).resolve().parent.parent / "routes" / "history_routes.py" +SRC = Path(__file__).resolve().parent.parent / "routes" / "history" / "history_routes.py" def _function_source(src_text, name): diff --git a/tests/test_history_order_by_timestamp_regression.py b/tests/test_history_order_by_timestamp_regression.py index 3fb2922a2..14e358928 100644 --- a/tests/test_history_order_by_timestamp_regression.py +++ b/tests/test_history_order_by_timestamp_regression.py @@ -25,7 +25,7 @@ from sqlalchemy.orm import sessionmaker from core.database import Base, ChatMessage as DbChatMessage, Session as DbSession -HISTORY_ROUTES = Path(__file__).resolve().parent.parent / "routes" / "history_routes.py" +HISTORY_ROUTES = Path(__file__).resolve().parent.parent / "routes" / "history" / "history_routes.py" def test_chatmessage_model_has_timestamp_not_created_at(): diff --git a/tests/test_history_routes_shim.py b/tests/test_history_routes_shim.py new file mode 100644 index 000000000..417a09f0d --- /dev/null +++ b/tests/test_history_routes_shim.py @@ -0,0 +1,43 @@ +"""Regression test for the history route shim (slice 2d, #4082/#4071). + +The backward-compat shim at ``routes/history_routes.py`` uses ``sys.modules`` +replacement so the legacy import path and the canonical ``routes.history.*`` +path resolve to the *same* module object. This is required because +``test_history_compact_tool_calls.py`` and ``test_fork_session_metadata.py`` +do ``import routes.history_routes as history_routes`` followed by +``monkeypatch.setattr(history_routes, "_verify_session_owner", ...)`` — for +those patches to take effect at runtime, the legacy module object and the +canonical one must be identical. This test pins that contract. +""" + +import importlib + +import routes.history_routes as _shim_history # noqa: F401 + + +def test_legacy_and_canonical_history_module_are_same_object(): + """``import routes.history_routes`` must alias the canonical module.""" + legacy = importlib.import_module("routes.history_routes") + canonical = importlib.import_module("routes.history.history_routes") + assert legacy is canonical, ( + "routes.history_routes shim must resolve to the canonical " + "routes.history.history_routes module object" + ) + + +def test_monkeypatch_via_legacy_alias_reaches_canonical(monkeypatch): + """Patching through the legacy alias must reach the canonical module. + + Several history tests do ``import routes.history_routes as history_routes`` + followed by ``monkeypatch.setattr(history_routes, "_verify_session_owner", + ...)``. For that to take effect at runtime, the legacy module object and the + canonical one must be identical. + """ + legacy = importlib.import_module("routes.history_routes") + canonical = importlib.import_module("routes.history.history_routes") + + sentinel = object() + monkeypatch.setattr(legacy, "setup_history_routes", sentinel) + assert canonical.setup_history_routes is sentinel, ( + "monkeypatch via legacy alias did not reach the canonical module" + ) diff --git a/tests/test_model_helper_owner_scope.py b/tests/test_model_helper_owner_scope.py index 4612fa363..44f2c31df 100644 --- a/tests/test_model_helper_owner_scope.py +++ b/tests/test_model_helper_owner_scope.py @@ -34,7 +34,7 @@ def test_task_parse_resolves_with_owner_scope(): def test_history_compact_resolves_with_owner_scope(): - body = _function_source("routes/history_routes.py", "compact_session") + body = _function_source("routes/history/history_routes.py", "compact_session") assert "owner = effective_user(request)" in body assert 'resolve_endpoint("utility", owner=owner or None)' in body From 5c16d39e9101a00b304b85e458665f3d4f0f11c1 Mon Sep 17 00:00:00 2001 From: badgerbees <93577481+badgerbees@users.noreply.github.com> Date: Sat, 4 Jul 2026 20:44:46 +0800 Subject: [PATCH 19/32] fix(calendar): honor list_events date range aliases (#3283) * fix(calendar): honor list_events date range aliases * fix(calendar): reject partially resolved loose range queries --- src/agent_loop.py | 2 +- src/tool_schemas.py | 4 +-- src/tools/calendar.py | 13 +++++++-- tests/test_calendar_list_range_aliases.py | 26 ++++++++++++++++++ tests/test_calendar_update_event_tz.py | 33 +++++++++++++++++++++++ 5 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/agent_loop.py b/src/agent_loop.py index e3fa1f0b6..5a4d1dbd7 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -469,7 +469,7 @@ Bulk delete/archive/mark emails. Use this for "delete all those" after listing e {"action": "create_event", "summary": "", "dtstart": ""} ``` Calendar event management (CalDAV). Actions: `list_events`, `create_event`, `update_event`, `delete_event`, `list_calendars`. \ -For `list_events`: {start?, end?, calendar?}; prefer `start`/`end` for the range, though start_date/end_date and from/to aliases are accepted. \ +For `list_events`: {action: "list_events", start: "YYYY-MM-DDT00:00:00", end: "YYYY-MM-DDT00:00:00", calendar?}; resolve month/week phrases yourself from the Current date and time context and do not pass a loose `query` field. Prefer `start`/`end`; start_time/end_time, start_date/end_date, and from/to aliases are accepted. \ For `create_event`: {summary, dtstart, dtend?, duration?, calendar?, location?, description?, reminder_minutes?, rrule?}. \ For `update_event`: {uid, summary?, dtstart?, dtend?, all_day?, location?, description?, event_type?, importance?, rrule?}. Pass `rrule: ""` to remove recurrence and make a repeating event a single event. \ `dtstart` accepts natural language ("tomorrow at 1pm", "in 2 hours", "next monday 9am") or ISO ("2026-05-12T13:00:00"). \ diff --git a/src/tool_schemas.py b/src/tool_schemas.py index 2006be9cd..a06fe352d 100644 --- a/src/tool_schemas.py +++ b/src/tool_schemas.py @@ -556,8 +556,8 @@ FUNCTION_TOOL_SCHEMAS = [ "uid": {"type": "string", "description": "Event UID (for update/delete)"}, "calendar_href": {"type": "string", "description": "Specific calendar URL (optional; defaults to first calendar)"}, "calendar": {"type": "string", "description": "Filter list_events by calendar name or href"}, - "start": {"type": "string", "description": "list_events range start (ISO datetime); defaults to today. Prefer start; backend also accepts start_date, range_start, from, dtstart, since."}, - "end": {"type": "string", "description": "list_events range end (ISO datetime); defaults to +14 days. Prefer end; backend also accepts end_date, range_end, to, dtend, until."}, + "start": {"type": "string", "description": "list_events range start (ISO datetime). Use this for month/week requests after resolving the date range; do not pass a loose query string. Prefer start; backend also accepts start_time, start_date, range_start, from, dtstart, since."}, + "end": {"type": "string", "description": "list_events range end (ISO datetime). Use this for month/week requests after resolving the date range; defaults to +14 days only when no range is requested. Prefer end; backend also accepts end_time, end_date, range_end, to, dtend, until."}, "event_type": {"type": "string", "description": "Tag / category for the event. Common values: work, personal, health, travel, meal, social, admin, other. Aliases accepted: tag, category, type."}, "importance": {"type": "string", "enum": ["low", "normal", "high", "critical"], "description": "Priority level (defaults to 'normal')"}, "reminder_minutes": {"type": "integer", "description": "For create_event: create an Odysseus reminder this many minutes before the event, e.g. 5 for 'reminder 5 min before'."}, diff --git a/src/tools/calendar.py b/src/tools/calendar.py index c1ebc43c8..d442f2a42 100644 --- a/src/tools/calendar.py +++ b/src/tools/calendar.py @@ -208,11 +208,20 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: elif action == "list_events": try: start_raw = _first_nonempty_arg( - "start", "start_date", "range_start", "from", "dtstart", "since" + "start", "start_time", "start_date", "range_start", "from", "dtstart", "since" ) end_raw = _first_nonempty_arg( - "end", "end_date", "range_end", "to", "dtend", "until" + "end", "end_time", "end_date", "range_end", "to", "dtend", "until" ) + query_raw = args.get("query") or args.get("date_range") or args.get("range") + if query_raw and (not start_raw or not end_raw): + return { + "error": ( + "list_events needs explicit start/end ISO datetimes; " + f"resolve the requested range ({query_raw!r}) and call manage_calendar again." + ), + "exit_code": 1, + } if start_raw: start_dt = _parse_dt(start_raw) else: diff --git a/tests/test_calendar_list_range_aliases.py b/tests/test_calendar_list_range_aliases.py index 669c8e009..cac16f60f 100644 --- a/tests/test_calendar_list_range_aliases.py +++ b/tests/test_calendar_list_range_aliases.py @@ -78,3 +78,29 @@ async def test_list_events_honors_range_aliases(start_key, end_key): summaries = [event["summary"] for event in res["events"]] assert summaries == ["Late June planning"] assert "between 2126-06-01 and 2126-07-01" in res["response"] + +async def test_list_events_rejects_partial_loose_range(): + from src.tool_implementations import do_manage_calendar + + owner = "calendar-partial-" + uuid.uuid4().hex[:8] + + # Partial: has query and start, but no end + res = await do_manage_calendar(json.dumps({ + "action": "list_events", + "query": "July", + "start_time": "2126-07-01T00:00:00Z", + }), owner=owner) + + assert res.get("exit_code", 1) == 1, res + assert "list_events needs explicit start/end" in res.get("error", "") + + # Partial: has query and end, but no start + res2 = await do_manage_calendar(json.dumps({ + "action": "list_events", + "query": "July", + "end_time": "2126-07-31T23:59:59Z", + }), owner=owner) + + assert res2.get("exit_code", 1) == 1, res2 + assert "list_events needs explicit start/end" in res2.get("error", "") + diff --git a/tests/test_calendar_update_event_tz.py b/tests/test_calendar_update_event_tz.py index 1ebbfce56..711aaa3e3 100644 --- a/tests/test_calendar_update_event_tz.py +++ b/tests/test_calendar_update_event_tz.py @@ -78,3 +78,36 @@ async def test_update_event_dtstart_anchored_to_user_tz(tokyo_offset): assert bool(ev.is_utc) is True finally: db.close() + + +async def test_list_events_accepts_start_time_end_time_aliases(): + from src.tool_implementations import do_manage_calendar + + owner = "list-" + uuid.uuid4().hex[:6] + created = await do_manage_calendar(json.dumps({ + "action": "create_event", + "summary": "July planning", + "dtstart": "2026-07-15T12:00:00Z", + "dtend": "2026-07-15T13:00:00Z", + }), owner=owner) + assert created.get("exit_code", 0) == 0, created + + listed = await do_manage_calendar(json.dumps({ + "action": "list_events", + "start_time": "2026-07-01T00:00:00Z", + "end_time": "2026-08-01T00:00:00Z", + }), owner=owner) + assert listed.get("exit_code", 0) == 0, listed + assert "between 2026-07-01 and 2026-08-01" in listed["response"] + assert [event["summary"] for event in listed["events"]] == ["July planning"] + + +async def test_list_events_query_without_range_does_not_default_to_two_weeks(): + from src.tool_implementations import do_manage_calendar + + listed = await do_manage_calendar(json.dumps({ + "action": "list_events", + "query": "July", + }), owner="list-" + uuid.uuid4().hex[:6]) + assert listed.get("exit_code") == 1, listed + assert "explicit start/end" in listed["error"] From d3faa00aaa1e8509bb0d28f5828fb640075e1d0f Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Sat, 4 Jul 2026 21:22:25 +0530 Subject: [PATCH 20/32] fix(security): match grep's rg sensitive-file exclusions case-insensitively (#5189) The grep tool's ripgrep fast-path excluded deny-listed key files with `--glob "!**"` for each entry in _SENSITIVE_FILE_PATTERNS. ripgrep's --glob is case-sensitive, so on a case-insensitive filesystem (Windows, default macOS) a key stored under a case variant of its name (ID_RSA, Known_Hosts, Authorized_Keys) is the same file on disk but slips past the lowercase exclusion, and ripgrep returns its contents. Those names are non-dotfiles, so ripgrep's default hidden-file skipping does not cover them either. The Python fallback already blocks them via the case-folded _is_sensitive_path (#5097), so the two paths disagreed. Switch the sensitive-pattern exclusions to --iglob so they match case-insensitively, mirroring _is_sensitive_path. Add a regression test that seeds ID_RSA and Known_Hosts and asserts grep returns ordinary matches but not the key contents. --- src/agent_tools/filesystem_tools.py | 6 +++++- tests/test_code_nav_tools.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/agent_tools/filesystem_tools.py b/src/agent_tools/filesystem_tools.py index 573d30db3..52dcb60f8 100644 --- a/src/agent_tools/filesystem_tools.py +++ b/src/agent_tools/filesystem_tools.py @@ -424,8 +424,12 @@ class GrepTool: cmd.append("--ignore-case") if glob_pat: cmd += ["--glob", glob_pat] + # --iglob (not --glob) so the exclusion is case-insensitive: + # on a case-insensitive filesystem "ID_RSA"/"Known_Hosts" + # resolve to the same secret as their lowercase forms, and the + # Python fallback below already folds case via _is_sensitive_path. for _pat in _SENSITIVE_FILE_PATTERNS: - cmd += ["--glob", f"!*{_pat}*"] + cmd += ["--iglob", f"!*{_pat}*"] for _d in _CODENAV_SKIP_DIRS: cmd += ["--glob", f"!**/{_d}/**"] cmd += ["--regexp", pattern, root] diff --git a/tests/test_code_nav_tools.py b/tests/test_code_nav_tools.py index d02840e18..5be50220a 100644 --- a/tests/test_code_nav_tools.py +++ b/tests/test_code_nav_tools.py @@ -91,6 +91,30 @@ def test_grep_python_fallback_when_no_rg(repo, monkeypatch): assert ".git/config" not in r["output"] +@pytest.mark.skipif(shutil.which("rg") is None, reason="targets the ripgrep fast-path") +def test_grep_skips_case_variant_sensitive_files_rg(repo): + """The rg fast-path must exclude deny-listed key files case-insensitively. + + A file whose name is a case variant of a sensitive pattern (e.g. ID_RSA vs + id_rsa, Known_Hosts vs known_hosts) points at the same secret on a + case-insensitive filesystem, so grep must not return its contents. The + Python fallback already folds case via _is_sensitive_path; a plain --glob + exclusion is case-sensitive, so it would leak these — this pins the rg path. + """ + token = "GREPSECRET_TOKEN_ZZZ" + with open(os.path.join(repo, "notes.txt"), "w") as f: + f.write(f"see {token}\n") + with open(os.path.join(repo, "ID_RSA"), "w") as f: + f.write(f"PRIVATE {token}\n") + with open(os.path.join(repo, "Known_Hosts"), "w") as f: + f.write(f"host {token}\n") + r = _run("grep", f'{{"pattern": "{token}", "path": "{repo}"}}') + assert r["exit_code"] == 0 + assert "notes.txt" in r["output"] # ordinary matches still returned + assert "ID_RSA" not in r["output"] # case-variant key excluded + assert "Known_Hosts" not in r["output"] + + # ── glob ────────────────────────────────────────────────────────────────── def test_glob_py(repo): From 3dd031c139ea8e97ec0a818df459d47ee8a13e99 Mon Sep 17 00:00:00 2001 From: Wes Huber Date: Sat, 4 Jul 2026 08:58:14 -0700 Subject: [PATCH 21/32] fix(security): validate integration api_call URLs with the outbound SSRF guard (#5145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit execute_api_call — reachable by the LLM through the api_call agent tool — joined the integration's user-configured base_url with an LLM-controlled path and requested it with no IP validation, so a base_url (or a hostname resolving) into the metadata range (169.254.169.254) was fetched server-side with the integration's auth headers attached. Run check_outbound_url on the joined URL before connecting, matching the gallery endpoint, embeddings, CardDAV, and reminder webhook surfaces. Link-local/metadata is always rejected; INTEGRATION_API_BLOCK_PRIVATE_IPS=true also blocks RFC-1918/loopback. Private stays allowed by default because LAN integrations (Home Assistant, Miniflux, ntfy) are the primary use case. The truncation-test helpers stub the guard open because their api.example.com fixture host does not resolve and the guard fails closed on DNS errors. Fixes #5143 Co-authored-by: Claude Fable 5 --- src/integrations.py | 16 +++ tests/test_integration_api_call_ssrf.py | 99 +++++++++++++++++++ .../test_integrations_api_call_truncation.py | 6 ++ 3 files changed, 121 insertions(+) create mode 100644 tests/test_integration_api_call_ssrf.py diff --git a/src/integrations.py b/src/integrations.py index 3b2b88859..3cb53a415 100644 --- a/src/integrations.py +++ b/src/integrations.py @@ -394,6 +394,22 @@ async def execute_api_call( return {"error": "Path must not contain a fragment", "exit_code": 1} url = _join_integration_url(base_url, path) + + # SSRF guard — same check used by the gallery endpoint, embeddings, + # CardDAV, and the reminder webhook sender. Link-local / metadata + # addresses (169.254.x.x — the cloud credential-exfil vector) are always + # rejected; INTEGRATION_API_BLOCK_PRIVATE_IPS=true also blocks RFC-1918 / + # loopback for locked-down deployments. Private stays allowed by default + # because LAN integrations (Home Assistant, Miniflux, ntfy) are the + # primary use case. + from src.url_safety import check_outbound_url + block_private = os.getenv( + "INTEGRATION_API_BLOCK_PRIVATE_IPS", "false" + ).lower() == "true" + ok, reason = check_outbound_url(url, block_private=block_private) + if not ok: + return {"error": f"URL rejected: {reason}", "exit_code": 1} + method = method.upper() # Build headers diff --git a/tests/test_integration_api_call_ssrf.py b/tests/test_integration_api_call_ssrf.py new file mode 100644 index 000000000..53dc671c5 --- /dev/null +++ b/tests/test_integration_api_call_ssrf.py @@ -0,0 +1,99 @@ +"""Regression: execute_api_call must run the outbound SSRF guard. + +The api_call agent tool lets the LLM drive HTTP requests against a +user-configured integration base_url. Before this guard, a base_url (or a +hostname resolving) to the cloud metadata range was requested server-side +with the integration's auth headers attached. execute_api_call now validates +the joined URL with src.url_safety.check_outbound_url before connecting: +link-local/metadata is always rejected; RFC-1918/loopback only when +INTEGRATION_API_BLOCK_PRIVATE_IPS=true (LAN integrations are the primary +use case, so private stays allowed by default). +""" +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src import integrations + + +def _integration(base_url): + return { + "id": "test_integ", + "name": "TestInteg", + "enabled": True, + "base_url": base_url, + "auth_type": "bearer", + "api_key": "secret-token", + "auth_header": "", + "auth_param": "", + "description": "", + "preset": "", + } + + +async def _call(base_url, path="/items"): + resp = MagicMock() + resp.status_code = 200 + resp.headers = {"content-type": "application/json"} + resp.json.return_value = {"ok": True} + resp.text = '{"ok": true}' + + client = AsyncMock() + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=None) + client.request = AsyncMock(return_value=resp) + + with ( + patch.object(integrations, "_find_integration", + return_value=_integration(base_url)), + patch("httpx.AsyncClient", return_value=client), + ): + result = await integrations.execute_api_call("test_integ", "GET", path) + return result, client + + +@pytest.mark.asyncio +async def test_metadata_ip_base_url_is_rejected_without_requesting(): + result, client = await _call("http://169.254.169.254") + + assert result["exit_code"] == 1 + assert "rejected" in result["error"].lower() + client.request.assert_not_called() + + +@pytest.mark.asyncio +async def test_hostname_resolving_to_metadata_ip_is_rejected(monkeypatch): + """DNS-based variant: an innocuous-looking hostname that resolves into + the link-local range must be caught by the resolver check.""" + monkeypatch.setattr("src.url_safety._default_resolver", + lambda host: ["169.254.169.254"]) + result, client = await _call("http://internal.attacker.example") + + assert result["exit_code"] == 1 + assert "rejected" in result["error"].lower() + client.request.assert_not_called() + + +@pytest.mark.asyncio +async def test_public_ip_base_url_still_requests(): + # Public literal — no DNS involved. + result, client = await _call("http://93.184.216.34") + + assert result.get("exit_code") == 0 + client.request.assert_called_once() + + +@pytest.mark.asyncio +async def test_private_base_url_allowed_by_default_blocked_with_knob(monkeypatch): + # Local-first default: LAN integrations (Home Assistant etc.) must work. + monkeypatch.delenv("INTEGRATION_API_BLOCK_PRIVATE_IPS", raising=False) + result, client = await _call("http://192.168.1.50") + assert result.get("exit_code") == 0 + client.request.assert_called_once() + + # Locked-down deployments opt in to a full private/loopback block. + monkeypatch.setenv("INTEGRATION_API_BLOCK_PRIVATE_IPS", "true") + result, client = await _call("http://192.168.1.50") + assert result["exit_code"] == 1 + assert "rejected" in result["error"].lower() + client.request.assert_not_called() diff --git a/tests/test_integrations_api_call_truncation.py b/tests/test_integrations_api_call_truncation.py index 97af90521..bf1ec7d05 100644 --- a/tests/test_integrations_api_call_truncation.py +++ b/tests/test_integrations_api_call_truncation.py @@ -83,6 +83,9 @@ async def _call(json_data, status=200): with ( patch.object(integrations, "_find_integration", return_value=DUMMY_INTEGRATION), patch("httpx.AsyncClient", return_value=mock_client), + # api.example.com doesn't resolve; the SSRF guard would fail closed. + # These tests are about truncation, so stub the guard open. + patch("src.url_safety.check_outbound_url", return_value=(True, "ok")), ): return await integrations.execute_api_call("test_integ", "GET", "/items") @@ -98,6 +101,9 @@ async def _call_with_integration(integration, path="/items"): with ( patch.object(integrations, "_find_integration", return_value=integration), patch("httpx.AsyncClient", return_value=mock_client), + # api.example.com doesn't resolve; the SSRF guard would fail closed. + # These tests are about URL joining, so stub the guard open. + patch("src.url_safety.check_outbound_url", return_value=(True, "ok")), ): result = await integrations.execute_api_call("test_integ", "GET", path) return result, mock_client From 6114ef0d6d13d4b78977a481b99445802eefe3bc Mon Sep 17 00:00:00 2001 From: Wes Huber Date: Sat, 4 Jul 2026 09:03:38 -0700 Subject: [PATCH 22/32] fix(security): pin webhook delivery to the SSRF-validated IP (DNS rebinding) (#5147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_webhook_url resolves the host to accept/reject, but the delivery connect (httpx.AsyncClient.post) re-resolved independently — a DNS record flipping between the two lookups (rebinding) could slip an internal IP (127.0.0.1 / 169.254.169.254 / LAN) past the check and receive the signed payload. The module docstring already flagged this as only a "partial defense". Resolve + validate once via _validated_public_ips, then pin the delivery TCP connect to that approved IP with an async _PinnedAsyncTransport built on the public httpcore/httpx APIs (mirrors the sync search-fetch pin from #704). The URL, Host header, and TLS SNI are unchanged, so certificate validation and vhost routing still target the original hostname; only the socket destination is pinned. Delivery now uses a per-request pinned client instead of one shared client, so close() is a no-op kept for API compatibility. Adds end-to-end tests that drive the real transport against loopback servers, proving the connect follows the pin rather than re-resolving the URL host. Fixes #5146 Co-authored-by: Claude Fable 5 --- src/webhook_manager.py | 157 +++++++++++++++++++++++- tests/test_webhook_dns_rebinding_pin.py | 156 +++++++++++++++++++++++ tests/test_webhook_ssrf_resilience.py | 29 +++-- 3 files changed, 324 insertions(+), 18 deletions(-) create mode 100644 tests/test_webhook_dns_rebinding_pin.py diff --git a/src/webhook_manager.py b/src/webhook_manager.py index af28fe2a7..ca3e454df 100644 --- a/src/webhook_manager.py +++ b/src/webhook_manager.py @@ -7,10 +7,12 @@ import ipaddress import json import logging import re +import ssl from datetime import datetime, timezone from typing import Optional from urllib.parse import urlparse +import httpcore import httpx from src.database import SessionLocal, Webhook @@ -125,6 +127,128 @@ def validate_webhook_url(url: str) -> str: return url +def _validated_public_ips(url: str) -> list: + """Resolve *url*'s host and return its IPs, raising ValueError if any is + private/internal. + + ``validate_webhook_url`` resolves the host to decide accept/reject, but the + subsequent ``httpx`` connect re-resolves independently — so a DNS record + that flips between the two lookups (rebinding) can slip an internal IP past + the check. Callers pin the delivery connection to the IP this function + returns, closing that TOCTOU. Fail closed: an unresolvable or partly-private + result raises rather than returning a usable IP. + """ + parsed = urlparse(url) + hostname = (parsed.hostname or "").strip() + if not hostname: + raise ValueError("URL must have a hostname") + try: + literal = ipaddress.ip_address(hostname) + except ValueError: + literal = None + if literal is not None: + if _ip_is_private(literal): + raise ValueError("URL must not point to private/internal addresses") + return [literal] + addrs = _resolve_hostname_ips(hostname) + if not addrs or any(_ip_is_private(a) for a in addrs): + raise ValueError("URL must not point to private/internal addresses") + return addrs + + +# httpcore raises its own exception hierarchy; map the ones a simple POST can +# surface back to their httpx equivalents so callers' `except httpx.*` blocks +# (and sanitize_error) behave exactly as they did with the default transport. +_HTTPCORE_TO_HTTPX_EXC = { + httpcore.ConnectError: httpx.ConnectError, + httpcore.ConnectTimeout: httpx.ConnectTimeout, + httpcore.NetworkError: httpx.NetworkError, + httpcore.PoolTimeout: httpx.PoolTimeout, + httpcore.ProtocolError: httpx.ProtocolError, + httpcore.ReadError: httpx.ReadError, + httpcore.ReadTimeout: httpx.ReadTimeout, + httpcore.RemoteProtocolError: httpx.RemoteProtocolError, + httpcore.TimeoutException: httpx.TimeoutException, + httpcore.WriteError: httpx.WriteError, + httpcore.WriteTimeout: httpx.WriteTimeout, +} + + +class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend): + """Async network backend that routes every TCP connect to a fixed IP. + + httpcore derives TLS SNI and the ``Host`` header from the request URL, not + from the connect host, so pinning only the socket destination keeps + certificate validation and vhost routing pointed at the original hostname. + """ + + def __init__(self, ip: ipaddress._BaseAddress): + self._ip = str(ip) + self._real = httpcore.AnyIOBackend() + + async def connect_tcp(self, host, port, timeout=None, local_address=None, + socket_options=None): + return await self._real.connect_tcp( + self._ip, port, timeout, local_address, socket_options + ) + + async def connect_unix_socket(self, path, timeout=None, socket_options=None): + return await self._real.connect_unix_socket(path, timeout, socket_options) + + async def sleep(self, seconds: float) -> None: + return await self._real.sleep(seconds) + + +class _PinnedAsyncTransport(httpx.AsyncBaseTransport): + """httpx transport that pins the TCP connect to a pre-resolved public IP. + + Uses only public ``httpcore`` / ``httpx`` APIs. The request URL is passed + through unchanged (Host + SNI stay the original hostname); only the socket + destination is pinned, closing the DNS-rebinding TOCTOU between the SSRF + check and the connect. HTTP/1.1 only — webhook deliveries are small POSTs. + """ + + def __init__(self, ip: ipaddress._BaseAddress): + self._pool = httpcore.AsyncConnectionPool( + ssl_context=ssl.create_default_context(), + http1=True, + http2=False, + network_backend=_PinnedAsyncBackend(ip), + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + core_req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + try: + core_resp = await self._pool.handle_async_request(core_req) + content = b"".join([chunk async for chunk in core_resp.aiter_stream()]) + await core_resp.aclose() + except Exception as exc: + mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc)) + if mapped is not None: + raise mapped(str(exc)) from exc + raise + return httpx.Response( + status_code=core_resp.status, + headers=core_resp.headers, + content=content, + extensions=core_resp.extensions, + ) + + async def aclose(self) -> None: + await self._pool.aclose() + + def validate_events(events_str: str) -> str: """Validate comma-separated event names. Returns cleaned string.""" events = [e.strip() for e in events_str.split(",") if e.strip()] @@ -198,8 +322,11 @@ def sanitize_error(error: str, max_len: int = 200) -> str: class WebhookManager: def __init__(self, api_key_manager=None): - # Disable redirects to prevent SSRF via redirect chains - self._client = httpx.AsyncClient(timeout=10, follow_redirects=False) + # No shared client: each delivery builds a short-lived client whose + # transport is pinned to the SSRF-approved IP (see _deliver / + # _send_request), so a single reusable client can't be pointed at + # different pinned hosts. Redirects stay disabled on every delivery + # client to prevent SSRF via redirect chains. self._loop: Optional[asyncio.AbstractEventLoop] = None self._api_key_manager = api_key_manager # Strong references to in-flight fire-and-forget tasks. asyncio only @@ -262,11 +389,28 @@ class WebhookManager: decrypted = self._decrypt_secret(encrypted_secret) await self._deliver(webhook_id, url, decrypted, "webhook.test", {"message": "Test ping from Odysseus"}) + async def _send_request(self, url: str, body: str, headers: dict, + ip: ipaddress._BaseAddress) -> httpx.Response: + """POST *body* to *url* with the TCP connect pinned to *ip*. + + Overridable seam: tests replace this to avoid real sockets. Redirects + are disabled so a 3xx can't bounce the delivery to another host. + """ + transport = _PinnedAsyncTransport(ip) + async with httpx.AsyncClient( + timeout=10, follow_redirects=False, transport=transport, + ) as client: + return await client.post(url, content=body, headers=headers) + async def _deliver(self, webhook_id: str, url: str, secret: Optional[str], event: str, payload: dict): """Internal delivery. Never call directly from outside this class (use deliver_test).""" - # Re-validate URL at delivery time in case DB was tampered with + # Re-validate URL at delivery time in case DB was tampered with, and + # capture the exact IPs that passed the check so the connect can be + # pinned to one of them (closes the DNS-rebinding TOCTOU: the check + # below and the socket connect no longer resolve independently). try: validate_webhook_url(url) + pinned_ips = _validated_public_ips(url) except ValueError as e: logger.warning(f"Webhook {webhook_id} has invalid URL, skipping: {e}") return @@ -283,7 +427,7 @@ class WebhookManager: db = SessionLocal() try: - resp = await self._client.post(url, content=body, headers=headers) + resp = await self._send_request(url, body, headers, pinned_ips[0]) db.query(Webhook).filter(Webhook.id == webhook_id).update({ "last_triggered_at": _utcnow(), "last_status_code": resp.status_code, @@ -305,4 +449,7 @@ class WebhookManager: db.close() async def close(self): - await self._client.aclose() + # Delivery clients are per-request and closed via their async context + # manager, so there is no long-lived client to tear down here. Kept for + # API compatibility with callers (e.g. app shutdown). + return None diff --git a/tests/test_webhook_dns_rebinding_pin.py b/tests/test_webhook_dns_rebinding_pin.py new file mode 100644 index 000000000..144a19e96 --- /dev/null +++ b/tests/test_webhook_dns_rebinding_pin.py @@ -0,0 +1,156 @@ +"""Regression: webhook delivery must pin the TCP connect to the SSRF-approved IP. + +validate_webhook_url resolves the host to accept/reject, but the delivery +connect previously re-resolved independently — a DNS record flipping between +the two lookups (rebinding) could slip an internal IP past the check. _deliver +now resolves+validates once via _validated_public_ips and pins the connect to +that IP through _PinnedAsyncTransport. These tests drive the real transport +against local servers so the pin is exercised end-to-end, not mocked away. +""" +import asyncio +import http.server +import ipaddress +import socketserver +import threading + +import pytest + +from tests.helpers.import_state import clear_module, preserve_import_state + +import os +import sys +from unittest.mock import patch + +with patch.dict(os.environ, {"DATABASE_URL": "sqlite:///:memory:"}), \ + preserve_import_state("src.database", "core.database"): + clear_module("src.database") + _core_database = sys.modules.get("core.database") + if _core_database is not None and not getattr(_core_database, "__file__", None): + del sys.modules["core.database"] + import src.webhook_manager as wm + + +# --------------------------------------------------------------------------- +# _validated_public_ips +# --------------------------------------------------------------------------- + +def test_validated_public_ips_rejects_metadata_literal(): + with pytest.raises(ValueError): + wm._validated_public_ips("http://169.254.169.254/") + + +def test_validated_public_ips_rejects_loopback_literal(): + with pytest.raises(ValueError): + wm._validated_public_ips("http://127.0.0.1/") + + +def test_validated_public_ips_returns_public_literal(): + ips = wm._validated_public_ips("http://93.184.216.34/") + assert ips == [ipaddress.ip_address("93.184.216.34")] + + +def test_validated_public_ips_rejects_hostname_resolving_private(monkeypatch): + # Rebinding shape: a hostname that (now) resolves into loopback space. + monkeypatch.setattr(wm, "_resolve_hostname_ips", + lambda h: [ipaddress.ip_address("127.0.0.1")]) + with pytest.raises(ValueError): + wm._validated_public_ips("http://evil.rebind.example/") + + +# --------------------------------------------------------------------------- +# End-to-end: the pinned transport actually routes to the pinned IP +# --------------------------------------------------------------------------- + +def _serve(handler): + srv = socketserver.TCPServer(("127.0.0.1", 0), handler) + port = srv.server_address[1] + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv, port + + +def test_pinned_transport_connects_to_pinned_ip(): + """A request whose URL host is a throwaway hostname is still delivered to + the pinned loopback IP — proving the socket destination comes from the pin, + not from resolving the URL host.""" + hits = [] + + class _Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + length = int(self.headers.get("Content-Length", 0)) + self.rfile.read(length) + hits.append(self.path) + self.send_response(204) + self.end_headers() + + def log_message(self, *a): + pass + + srv, port = _serve(_Handler) + try: + ip = ipaddress.ip_address("127.0.0.1") + transport = wm._PinnedAsyncTransport(ip) + + async def go(): + async with __import__("httpx").AsyncClient( + transport=transport, timeout=5, follow_redirects=False, + ) as client: + # Host "unresolvable.invalid" would never resolve; the pin is + # what makes this reach the loopback server on `port`. + return await client.post( + f"http://unresolvable.invalid:{port}/hook", content=b"{}", + ) + + resp = asyncio.run(go()) + assert resp.status_code == 204 + assert hits == ["/hook"] + finally: + srv.shutdown() + + +def test_deliver_pins_to_validated_ip_end_to_end(monkeypatch): + """Full _deliver path: a hostname that validation resolves to loopback is + pinned to loopback and the local server receives the signed POST.""" + received = {} + + class _Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + length = int(self.headers.get("Content-Length", 0)) + received["body"] = self.rfile.read(length) + received["event"] = self.headers.get("X-Odysseus-Event") + self.send_response(200) + self.end_headers() + + def log_message(self, *a): + pass + + srv, port = _serve(_Handler) + + class _Query: + def filter(self, *a, **k): return self + def update(self, values): return None + + class _Db: + def query(self, _m): return _Query() + def commit(self): pass + def rollback(self): pass + def close(self): pass + + # Make both the validation resolve and the pin target loopback, and treat + # loopback as allowed for this test (production blocks it — here we only + # want to prove the pin routes to the validated IP). + monkeypatch.setattr(wm, "SessionLocal", lambda: _Db()) + monkeypatch.setattr(wm, "_is_private_url", lambda url: False) + monkeypatch.setattr(wm, "_resolve_hostname_ips", + lambda h: [ipaddress.ip_address("127.0.0.1")]) + monkeypatch.setattr(wm, "_ip_is_private", lambda a: False) + + manager = wm.WebhookManager() + try: + asyncio.run(manager._deliver( + "hook-1", f"http://webhook.test:{port}/cb", "s3cret", + "webhook.test", {"ok": True}, + )) + assert received.get("event") == "webhook.test" + assert b'"ok": true' in received["body"] + finally: + srv.shutdown() diff --git a/tests/test_webhook_ssrf_resilience.py b/tests/test_webhook_ssrf_resilience.py index e02f17a25..ca82a7595 100644 --- a/tests/test_webhook_ssrf_resilience.py +++ b/tests/test_webhook_ssrf_resilience.py @@ -96,26 +96,29 @@ async def test_webhook_delivery_uses_naive_utc_timestamps(monkeypatch): class _Response: status_code = 204 - class _Client: - def __init__(self): - self.content = "" - - async def post(self, _url, content, headers): - self.content = content - assert headers["X-Odysseus-Event"] == "webhook.test" - return _Response() - db = _Db() - client = _Client() monkeypatch.setattr(wm, "SessionLocal", lambda: db) manager = wm.WebhookManager() - await manager._client.aclose() - manager._client = client + + # Replace the pinned-transport send seam so no real socket is opened. The + # public-IP literal below still exercises _validated_public_ips (which pins + # the connect); the captured content proves the body/headers are built. + captured = {} + + async def _fake_send(url, body, headers, ip): + captured["content"] = body + captured["ip"] = str(ip) + assert headers["X-Odysseus-Event"] == "webhook.test" + return _Response() + + monkeypatch.setattr(manager, "_send_request", _fake_send) await manager._deliver("hook-1", "http://93.184.216.34/", None, "webhook.test", {"ok": True}) - body = json.loads(client.content) + # The delivery must have pinned to the literal public IP from the URL. + assert captured["ip"] == "93.184.216.34" + body = json.loads(captured["content"]) payload_timestamp = datetime.fromisoformat(body["timestamp"]) assert payload_timestamp.tzinfo is None assert db.updates[0]["last_triggered_at"].tzinfo is None From 897e6950af4d0485e87c5e48c16487fc40b19588 Mon Sep 17 00:00:00 2001 From: Wes Huber Date: Sat, 4 Jul 2026 09:05:37 -0700 Subject: [PATCH 23/32] fix(security): apply the webhook SSRF guard to the reminder ntfy sender (#5142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The webhook branch of dispatch_reminder validates its target with check_outbound_url before posting; the ntfy branch posted to the integration's user-configured base_url with no check, so a base_url pointing at the metadata range (169.254.169.254) was fetched server-side — with the integration's Authorization header attached — every time a reminder fired. Run the same check (and honor the same REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS knob) before the post, surfacing rejections in ntfy_error exactly like the webhook branch does. LAN ntfy servers keep working by default, matching the project's local-first policy. Fixes #5141 Co-authored-by: Claude Fable 5 --- routes/note_routes.py | 21 ++++-- tests/test_reminder_ntfy_ssrf.py | 117 +++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 tests/test_reminder_ntfy_ssrf.py diff --git a/routes/note_routes.py b/routes/note_routes.py index 2a91fedf8..3d356f5c9 100644 --- a/routes/note_routes.py +++ b/routes/note_routes.py @@ -483,11 +483,22 @@ async def dispatch_reminder( api_key = intg.get("api_key", "") if api_key: hdrs["Authorization"] = f"Bearer {api_key}" - async with httpx.AsyncClient(timeout=10.0) as client: - resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs) - ntfy_sent = resp.is_success - if not ntfy_sent: - ntfy_error = f"ntfy returned HTTP {resp.status_code}" + # SSRF guard — same check (and env knob) as the webhook branch + # above: link-local / metadata addresses are always rejected; + # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918 + # so a ntfy base_url can't be pointed at internal services. + import os as _os + from src.url_safety import check_outbound_url as _chk + _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true" + _ok, _reason = _chk(f"{base}/{topic}", block_private=_block) + if not _ok: + ntfy_error = f"ntfy URL rejected: {_reason}" + else: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs) + ntfy_sent = resp.is_success + if not ntfy_sent: + ntfy_error = f"ntfy returned HTTP {resp.status_code}" else: ntfy_error = "No enabled ntfy integration" except Exception as e: diff --git a/tests/test_reminder_ntfy_ssrf.py b/tests/test_reminder_ntfy_ssrf.py new file mode 100644 index 000000000..40e16831b --- /dev/null +++ b/tests/test_reminder_ntfy_ssrf.py @@ -0,0 +1,117 @@ +"""Regression: the reminder ntfy sender must run the same SSRF guard as the +webhook sender. + +The webhook branch of dispatch_reminder validates its target with +src.url_safety.check_outbound_url before posting; the ntfy branch posted to +the integration's base_url with no check, so a base_url pointing at the cloud +metadata range (169.254.169.254) was fetched server-side — with the +integration's Authorization header attached — every time a reminder fired. +""" +import asyncio +from unittest.mock import MagicMock, patch + +import httpx + +from routes.note_routes import dispatch_reminder + + +def _ntfy_integration(base_url): + return [{ + "preset": "ntfy", + "enabled": True, + "base_url": base_url, + "api_key": "secret-token", + "name": "ntfy", + }] + + +def _settings(**extra): + return { + "reminder_channel": "ntfy", + "reminder_llm_synthesis": False, + "reminder_ntfy_topic": "reminders", + **extra, + } + + +class _SpyAsyncClient: + """Stands in for httpx.AsyncClient; records posts, returns success.""" + calls = [] + + def __init__(self, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + pass + + async def post(self, url, **kw): + _SpyAsyncClient.calls.append(url) + resp = MagicMock() + resp.is_success = True + resp.status_code = 200 + return resp + + +def _dispatch(): + return asyncio.run(dispatch_reminder( + "Title", "Body", note_id="", queue_browser=True, + settings_override=_settings(), + )) + + +def test_metadata_ip_ntfy_base_url_is_rejected_and_not_fetched(): + _SpyAsyncClient.calls = [] + with ( + patch("src.integrations.load_integrations", + return_value=_ntfy_integration("http://169.254.169.254")), + patch.object(httpx, "AsyncClient", _SpyAsyncClient), + ): + result = _dispatch() + + assert _SpyAsyncClient.calls == [], "metadata address must never be fetched" + assert result["ntfy_sent"] is False + assert "rejected" in result["ntfy_error"].lower() + + +def test_public_ntfy_base_url_still_sends(): + _SpyAsyncClient.calls = [] + with ( + # 93.184.216.34 is a public literal — no DNS resolution involved. + patch("src.integrations.load_integrations", + return_value=_ntfy_integration("http://93.184.216.34")), + patch.object(httpx, "AsyncClient", _SpyAsyncClient), + ): + result = _dispatch() + + assert _SpyAsyncClient.calls == ["http://93.184.216.34/reminders"] + assert result["ntfy_sent"] is True + assert result["ntfy_error"] == "" + + +def test_private_ntfy_base_url_blocked_only_with_env_knob(monkeypatch): + # Default (local-first): a LAN ntfy server is a normal setup and must work. + _SpyAsyncClient.calls = [] + monkeypatch.delenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", raising=False) + with ( + patch("src.integrations.load_integrations", + return_value=_ntfy_integration("http://192.168.1.50")), + patch.object(httpx, "AsyncClient", _SpyAsyncClient), + ): + result = _dispatch() + assert result["ntfy_sent"] is True + + # Locked-down deployments: the same knob the webhook branch honors. + _SpyAsyncClient.calls = [] + monkeypatch.setenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "true") + with ( + patch("src.integrations.load_integrations", + return_value=_ntfy_integration("http://192.168.1.50")), + patch.object(httpx, "AsyncClient", _SpyAsyncClient), + ): + result = _dispatch() + assert _SpyAsyncClient.calls == [] + assert result["ntfy_sent"] is False + assert "rejected" in result["ntfy_error"].lower() From 439285e1b78f3f596989bc95cb1bbf2c2b02f9f7 Mon Sep 17 00:00:00 2001 From: Ohualtex <230626673+Ohualtex@users.noreply.github.com> Date: Sat, 4 Jul 2026 20:42:06 +0300 Subject: [PATCH 24/32] fix(search): extract non-ASCII capitalized names in _extract_entities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _extract_entities used the ASCII-only class [A-Z][a-zA-Z]+ to pull name entities from a query, so non-ASCII names were dropped ("İstanbul", "Zürich" yielded nothing) or shredded ("São Paulo" -> only "Paulo"), degrading query enhancement for non-English/accented searches. Match Unicode words and keep the alphabetic, uppercase-initial ones; ASCII behaviour (the word boundary already excludes camelCase mid-word capitals) is unchanged. --- services/search/query.py | 10 ++++++-- tests/test_search_query_unicode_names.py | 31 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 tests/test_search_query_unicode_names.py diff --git a/services/search/query.py b/services/search/query.py index 3bb398446..194610f38 100644 --- a/services/search/query.py +++ b/services/search/query.py @@ -34,8 +34,14 @@ def _extract_entities(query: str) -> Dict[str, List[str]]: cleaned = query if qtype: cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip() - for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned): - entities["names"].append(token) + # Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+ + # class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and + # "São" (shredded). Keep the ASCII behaviour — the word boundary already + # excludes camelCase mid-word capitals — by requiring an all-alphabetic + # token of length > 1 whose first character is uppercase. + for token in re.findall(r"\b\w+\b", cleaned): + if len(token) > 1 and token[0].isupper() and token.isalpha(): + entities["names"].append(token) for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned): entities["dates"].append(year) month_day_year = re.findall( diff --git a/tests/test_search_query_unicode_names.py b/tests/test_search_query_unicode_names.py new file mode 100644 index 000000000..104ba310f --- /dev/null +++ b/tests/test_search_query_unicode_names.py @@ -0,0 +1,31 @@ +"""Regression: _extract_entities must find non-ASCII capitalized names. + +The name extractor used the ASCII-only class [A-Z][a-zA-Z]+, so a query like +"İstanbul weather" or "Zürich hotels" yielded no name entities at all, and +"São Paulo" lost "São" — non-English/accented place and proper names were +silently dropped from query enhancement. Detection is now Unicode-aware; +ASCII behaviour (including camelCase mid-word capitals not counting as names) +is preserved. +""" +from services.search.query import _extract_entities + + +def _names(q): + return _extract_entities(q)["names"] + + +def test_non_ascii_names_are_extracted(): + assert "İstanbul" in _names("İstanbul weather") + assert "Zürich" in _names("Zürich hotels") + assert set(_names("trip to São Paulo")) >= {"São", "Paulo"} + + +def test_ascii_names_unchanged(): + assert _names("What did Alice do in 2024") == ["Alice"] + assert _names("news about OpenAI and Google") == ["OpenAI", "Google"] + + +def test_lowercase_camelcase_and_numbers_are_not_names(): + assert _names("the iphone price") == [] + assert _names("iPhone price") == [] # mid-word capital is not a name + assert _names("top 50 albums") == [] From e3750fcdcbdb9d97d28cc5dbd20e09d6812ddd9d Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:17:45 +0100 Subject: [PATCH 25/32] fix(security): make research path lookup CodeQL-friendly (#5129) * fix(security): make research path lookup CodeQL-friendly * fix(security): avoid duplicate research path scans * fix(research): preserve active completed spinoff query --- routes/research/research_routes.py | 124 ++++--- .../test_research_routes_path_confinement.py | 344 ++++++++++++++++-- 2 files changed, 397 insertions(+), 71 deletions(-) diff --git a/routes/research/research_routes.py b/routes/research/research_routes.py index b35c94b69..fdc650d95 100644 --- a/routes/research/research_routes.py +++ b/routes/research/research_routes.py @@ -21,23 +21,52 @@ from src.constants import DEEP_RESEARCH_DIR _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") -def _confine_research_path(session_id: str) -> Path: - """Return the resolved Path for session_id's JSON inside DEEP_RESEARCH_DIR. - - Validates the session ID format and asserts containment after symlink - expansion. Raises HTTPException(400) on format failures, traversal - attempts, absolute-path injection, and symlink escape so every caller - gets a safe, confined path with no extra validation needed. - """ +def _validate_session_id(session_id: str) -> str: if not _SESSION_ID_RE.fullmatch(session_id): - raise HTTPException(400, "Invalid session ID") - root = Path(DEEP_RESEARCH_DIR).resolve() - candidate = (root / f"{session_id}.json").resolve() + raise HTTPException(400, "Invalid session ID format") + return session_id + + +def _research_storage_root() -> Path: + return Path(DEEP_RESEARCH_DIR).resolve() + + +def _find_research_path(session_id: str) -> Path | None: + """Find a persisted research file without deriving its path from input.""" + expected_name = f"{_validate_session_id(session_id)}.json" + root = _research_storage_root() + for stored_path in root.glob("*.json"): + if stored_path.name != expected_name: + continue + resolved = stored_path.resolve() + try: + resolved.relative_to(root) + except ValueError: + return None + if not resolved.is_file(): + return None + return resolved + return None + + +def _require_research_path(session_id: str) -> Path: + path = _find_research_path(session_id) + if path is None: + raise HTTPException(404, "Research not found") + return path + + +def _find_owned_research_path(session_id: str, user: str) -> Path | None: + path = _find_research_path(session_id) + if path is None: + return None try: - candidate.relative_to(root) - except ValueError: - raise HTTPException(400, "Invalid session ID") - return candidate + owner = json.loads(path.read_text(encoding="utf-8")).get("owner") + except Exception: + return None + if owner != user: + return None + return path logger = logging.getLogger(__name__) @@ -192,10 +221,6 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: raise HTTPException(401, "Not authenticated") return user - def _validate_session_id(session_id: str) -> None: - if not _SESSION_ID_RE.fullmatch(session_id): - raise HTTPException(400, "Invalid session ID format") - def _owns_in_memory(session_id: str, user: str) -> bool: """Ownership check for an in-flight (in-memory) research task. Falls back to the on-disk JSON if the task has already finished.""" @@ -204,15 +229,32 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: return entry.get("owner", "") == user # Task no longer in memory — check the persisted JSON. try: - path = _confine_research_path(session_id) + return _find_owned_research_path(session_id, user) is not None except HTTPException: return False - if not path.exists(): - return False - try: - return json.loads(path.read_text(encoding="utf-8")).get("owner") == user - except Exception: - return False + + def _require_owned_or_active_research_path(session_id: str, user: str) -> Path | None: + """Validate ownership once and return the completed on-disk path. + + Active running research has no completed disk path yet. Completed + tasks can remain in _active_tasks after persistence, so prefer their + owned disk path when available. Completed disk lookups still reuse the + path after the ownership gate. + """ + entry = research_handler._active_tasks.get(session_id) + if entry is not None: + if entry.get("owner", "") != user: + raise HTTPException(404, "No research found for this session") + if entry.get("status") != "running": + path = _find_owned_research_path(session_id, user) + if path is not None: + return path + return None + + path = _find_owned_research_path(session_id, user) + if path is None: + raise HTTPException(404, "No research found for this session") + return path @router.get("/api/research/active") async def research_active(request: Request): @@ -270,9 +312,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: def _assert_owns_research(session_id: str, user: str) -> None: """404-not-403 ownership gate for a research session's on-disk JSON. Use BEFORE returning any data or mutating the file.""" - path = _confine_research_path(session_id) - if not path.exists(): - raise HTTPException(404, "Research not found") + path = _require_research_path(session_id) try: owner = json.loads(path.read_text(encoding="utf-8")).get("owner") except Exception: @@ -384,9 +424,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: summary, stats — used by the Library preview panel.""" user = _require_user(request) _validate_session_id(session_id) - path = _confine_research_path(session_id) - if not path.exists(): - raise HTTPException(404, "Research not found") + path = _require_research_path(session_id) try: data = json.loads(path.read_text(encoding="utf-8")) except Exception as e: @@ -401,9 +439,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Soft-archive / restore a research report (sets `archived` in its JSON).""" user = _require_user(request) _validate_session_id(session_id) - path = _confine_research_path(session_id) - if not path.exists(): - raise HTTPException(404, "Research not found") + path = _require_research_path(session_id) try: data = json.loads(path.read_text(encoding="utf-8")) if data.get("owner") != user: @@ -421,9 +457,9 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Delete a research result from disk.""" user = _require_user(request) _validate_session_id(session_id) - json_path = _confine_research_path(session_id) + json_path = _find_research_path(session_id) deleted = False - if json_path.exists(): + if json_path is not None: # SECURITY: verify ownership before letting the caller delete it. try: data = json.loads(json_path.read_text(encoding="utf-8")) @@ -579,12 +615,11 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: """Get research result without clearing it (for panel use).""" user = _require_user(request) _validate_session_id(session_id) - if not _owns_in_memory(session_id, user): - raise HTTPException(404, "No research found for this session") + owned_disk_path = _require_owned_or_active_research_path(session_id, user) result = research_handler.get_result(session_id) if result is None: - p = _confine_research_path(session_id) - if p.exists(): + p = owned_disk_path + if p is not None: d = json.loads(p.read_text(encoding="utf-8")) return { "result": d.get("result", ""), @@ -613,8 +648,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: # otherwise any authenticated user could spin off (and thereby read) # another user's report by guessing its session ID. Mirrors every other # endpoint in this file (see result_peek above). - if not _owns_in_memory(session_id, user): - raise HTTPException(404, "No research found for this session") + owned_disk_path = _require_owned_or_active_research_path(session_id, user) if session_manager is None: raise HTTPException(500, "session_manager not configured") @@ -623,8 +657,8 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: sources = research_handler.get_sources(session_id) or [] query = "" - path = _confine_research_path(session_id) - if path.exists(): + path = owned_disk_path + if path is not None: try: disk = json.loads(path.read_text(encoding="utf-8")) if not result: diff --git a/tests/test_research_routes_path_confinement.py b/tests/test_research_routes_path_confinement.py index 29c8a0715..ee19dcdbd 100644 --- a/tests/test_research_routes_path_confinement.py +++ b/tests/test_research_routes_path_confinement.py @@ -1,6 +1,6 @@ """Path-confinement regression tests for research routes. -Covers the CodeQL py/path-injection alert cluster (#552-#567) in +Covers the CodeQL py/path-injection alert cluster (#552-#567 and #594) in routes/research/research_routes.py: - _owns_in_memory disk fallback (alerts #552, #553) - _assert_owns_research (alerts #554, #555) @@ -20,7 +20,11 @@ import pytest from fastapi import HTTPException from routes.research_routes import setup_research_routes -from routes.research.research_routes import _confine_research_path +from routes.research.research_routes import ( + _find_owned_research_path, + _find_research_path, + _require_research_path, +) @pytest.fixture(autouse=True) @@ -58,15 +62,30 @@ def _research_handler(): # --------------------------------------------------------------------------- -# Helper-level tests — _confine_research_path +# Helper-level tests # --------------------------------------------------------------------------- -def test_confine_allows_valid_session_id(tmp_path, monkeypatch): +def test_find_returns_existing_trusted_research_path(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + expected = _write_research(data_dir, "rp-abc123de4567", owner="alice") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + assert _find_research_path("rp-abc123de4567") == expected.resolve() + + +def test_find_returns_none_for_missing_valid_session_id(tmp_path, monkeypatch): data_dir = tmp_path / "deep_research" data_dir.mkdir() monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) - path = _confine_research_path("rp-abc123de4567") - assert path == (data_dir / "rp-abc123de4567.json").resolve() + assert _find_research_path("rp-missing12345") is None + + +def test_require_returns_404_for_missing_valid_session_id(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + data_dir.mkdir() + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + with pytest.raises(HTTPException) as exc: + _require_research_path("rp-missing12345") + assert exc.value.status_code == 404 @pytest.mark.parametrize("bad_id", [ @@ -79,17 +98,45 @@ def test_confine_allows_valid_session_id(tmp_path, monkeypatch): "rp-bad.json", # dot not in allowed charset "a" * 129, # exceeds length limit ]) -def test_confine_rejects_bad_session_ids(tmp_path, monkeypatch, bad_id): - data_dir = tmp_path / "deep_research" - data_dir.mkdir() - monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) +def test_find_rejects_bad_session_ids_before_enumeration(monkeypatch, bad_id): + storage_root = MagicMock() + monkeypatch.setattr( + "routes.research.research_routes._research_storage_root", + MagicMock(return_value=storage_root), + ) with pytest.raises(HTTPException) as exc: - _confine_research_path(bad_id) + _find_research_path(bad_id) assert exc.value.status_code == 400 + storage_root.glob.assert_not_called() -def test_confine_rejects_symlink_escape(tmp_path, monkeypatch): - """A symlink inside DEEP_RESEARCH_DIR that resolves outside is rejected.""" +def test_find_matches_names_from_trusted_enumeration_without_joining_input( + tmp_path, monkeypatch +): + """Pin the CodeQL-friendly lookup: match a glob result, never root / input.""" + data_dir = tmp_path / "deep_research" + expected = _write_research(data_dir, "rp-abc123de4567", owner="alice").resolve() + + class EnumeratedRoot: + def glob(self, pattern): + assert pattern == "*.json" + return [expected] + + def __fspath__(self): + return str(data_dir.resolve()) + + def __truediv__(self, _other): + raise AssertionError("user-derived path segment was joined to root") + + monkeypatch.setattr( + "routes.research.research_routes._research_storage_root", + lambda: EnumeratedRoot(), + ) + assert _find_research_path("rp-abc123de4567") == expected + + +def test_find_ignores_symlink_escape(tmp_path, monkeypatch): + """A matching symlink that resolves outside is not a trusted file.""" data_dir = tmp_path / "deep_research" outside = tmp_path / "outside" data_dir.mkdir() @@ -102,9 +149,24 @@ def test_confine_rejects_symlink_escape(tmp_path, monkeypatch): link.symlink_to(target) except (AttributeError, NotImplementedError, OSError) as e: pytest.skip(f"symlinks unavailable: {e}") - with pytest.raises(HTTPException) as exc: - _confine_research_path("rp-linktest1234") - assert exc.value.status_code == 400 + assert _find_research_path("rp-linktest1234") is None + + + +def test_find_owned_returns_path_for_matching_owner(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + expected = _write_research(data_dir, "rp-ownedalice1", owner="alice") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + assert _find_owned_research_path("rp-ownedalice1", "alice") == expected.resolve() + + +def test_find_owned_returns_none_for_other_owner(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + _write_research(data_dir, "rp-ownedbybob12", owner="bob") + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + assert _find_owned_research_path("rp-ownedbybob12", "alice") is None # --------------------------------------------------------------------------- @@ -120,6 +182,24 @@ def test_detail_returns_data_for_owner(tmp_path): assert out["query"] == "valid query" +def test_detail_returns_404_for_missing_valid_id(): + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/detail/{session_id}", "GET") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id="rp-missing12345", request=_request("alice"))) + assert exc.value.status_code == 404 + + +def test_detail_hides_other_owners_research_with_404(tmp_path): + data_dir = tmp_path / "deep_research" + _write_research(data_dir, "rp-ownedbybob12", owner="bob") + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/detail/{session_id}", "GET") + with pytest.raises(HTTPException) as exc: + asyncio.run(target(session_id="rp-ownedbybob12", request=_request("alice"))) + assert exc.value.status_code == 404 + + # --------------------------------------------------------------------------- # Route-level tests — traversal and injection rejected # --------------------------------------------------------------------------- @@ -214,28 +294,76 @@ def test_detail_traversal_does_not_read_outside_file(tmp_path, monkeypatch): # Route-level symlink escape test # --------------------------------------------------------------------------- -def test_detail_rejects_symlink_escape(tmp_path, monkeypatch): - """research_detail rejects a confined-format ID whose JSON is a symlink to outside.""" +def _write_outside_symlink(tmp_path, session_id: str, data: dict): data_dir = tmp_path / "deep_research" outside_dir = tmp_path / "outside" data_dir.mkdir(parents=True) outside_dir.mkdir() - outside_file = outside_dir / "rp-linktest5678.json" - outside_file.write_text( - json.dumps({"owner": "alice", "result": "secret"}), encoding="utf-8" - ) - link = data_dir / "rp-linktest5678.json" + outside_file = outside_dir / f"{session_id}.json" + outside_file.write_text(json.dumps(data), encoding="utf-8") + link = data_dir / f"{session_id}.json" try: link.symlink_to(outside_file) except (AttributeError, NotImplementedError, OSError) as e: pytest.skip(f"symlinks unavailable: {e}") + return data_dir, outside_file + + +def test_detail_rejects_symlink_escape(tmp_path, monkeypatch): + """research_detail never reads a matching symlink outside the root.""" + data_dir, _ = _write_outside_symlink( + tmp_path, + "rp-linktest5678", + {"owner": "alice", "result": "secret"}, + ) monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) router = setup_research_routes(_research_handler()) target = _route(router, "/api/research/detail/{session_id}", "GET") with pytest.raises(HTTPException) as exc: asyncio.run(target(session_id="rp-linktest5678", request=_request("alice"))) - assert exc.value.status_code == 400 + assert exc.value.status_code == 404 + + +def test_archive_does_not_write_through_symlink_escape(tmp_path, monkeypatch): + data_dir, outside_file = _write_outside_symlink( + tmp_path, + "rp-linkarchive1", + {"owner": "alice", "archived": False}, + ) + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/{session_id}/archive", "POST") + with pytest.raises(HTTPException) as exc: + asyncio.run( + target( + session_id="rp-linkarchive1", + request=_request("alice"), + archived=True, + ) + ) + assert exc.value.status_code == 404 + assert json.loads(outside_file.read_text(encoding="utf-8"))["archived"] is False + + +def test_delete_does_not_unlink_symlink_escape(tmp_path, monkeypatch): + data_dir, outside_file = _write_outside_symlink( + tmp_path, + "rp-linkdelete12", + {"owner": "alice"}, + ) + link = data_dir / "rp-linkdelete12.json" + monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) + + router = setup_research_routes(_research_handler()) + target = _route(router, "/api/research/{session_id}", "DELETE") + out = asyncio.run( + target(session_id="rp-linkdelete12", request=_request("alice")) + ) + assert out == {"deleted": False} + assert link.is_symlink() + assert outside_file.exists() # --------------------------------------------------------------------------- @@ -243,14 +371,15 @@ def test_detail_rejects_symlink_escape(tmp_path, monkeypatch): # --------------------------------------------------------------------------- def test_owner_scoped_paths_stay_within_research_root(tmp_path, monkeypatch): - """Owner-scoped session IDs never produce paths outside DEEP_RESEARCH_DIR.""" + """Owner-scoped persisted files resolve within DEEP_RESEARCH_DIR.""" data_dir = tmp_path / "deep_research" data_dir.mkdir(parents=True) monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir)) root = data_dir.resolve() for session_id in ("rp-abc123456789", "rp-000000000001", "abc-xyz-123"): - path = _confine_research_path(session_id) + _write_research(data_dir, session_id, owner="alice") + path = _require_research_path(session_id) assert path.resolve().is_relative_to(root), ( f"{session_id!r} produced path outside research root: {path}" ) @@ -271,3 +400,166 @@ def test_spinoff_rejects_traversal(bad_id): with pytest.raises(HTTPException) as exc: asyncio.run(target(session_id=bad_id, request=_request("alice"))) assert exc.value.status_code == 400 + +def test_result_peek_uses_single_disk_lookup_for_completed_result(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + path = _write_research( + data_dir, + "rp-peeksingle1", + owner="alice", + result="saved result", + sources=["s1"], + raw_findings=["f1"], + category="security", + ).resolve() + + calls = [] + + def fake_find_owned(session_id, user): + calls.append((session_id, user)) + return path + + monkeypatch.setattr( + "routes.research.research_routes._find_owned_research_path", + fake_find_owned, + ) + + handler = _research_handler() + handler.get_result.return_value = None + router = setup_research_routes(handler) + target = _route(router, "/api/research/result-peek/{session_id}", "POST") + + out = asyncio.run(target(session_id="rp-peeksingle1", request=_request("alice"))) + + assert out["result"] == "saved result" + assert out["sources"] == ["s1"] + assert out["raw_findings"] == ["f1"] + assert out["category"] == "security" + assert calls == [("rp-peeksingle1", "alice")] + + +def test_spinoff_uses_single_disk_lookup_for_completed_result(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + path = _write_research( + data_dir, + "rp-spinsingle1", + owner="alice", + result="saved report", + sources=["s1", "s2"], + query="original query", + ).resolve() + + calls = [] + + def fake_find_owned(session_id, user): + calls.append((session_id, user)) + return path + + class FakeSession: + endpoint_url = "" + model = "" + headers = {} + + def __init__(self): + self.messages = [] + + def add_message(self, message): + self.messages.append(message) + + class FakeSessionManager: + def __init__(self): + self.created = None + + def get_session(self, session_id): + raise KeyError(session_id) + + def create_session(self, **kwargs): + self.created = FakeSession() + return self.created + + def save_sessions(self): + pass + + monkeypatch.setattr( + "routes.research.research_routes._find_owned_research_path", + fake_find_owned, + ) + monkeypatch.setattr( + "routes.research.research_routes.resolve_endpoint", + lambda *_args, **_kwargs: ("http://endpoint/v1", "model", {}), + ) + + handler = _research_handler() + handler.get_result.return_value = None + handler.get_sources.return_value = [] + session_manager = FakeSessionManager() + router = setup_research_routes(handler, session_manager=session_manager) + target = _route(router, "/api/research/spinoff/{session_id}", "POST") + + out = asyncio.run(target(session_id="rp-spinsingle1", request=_request("alice"))) + + assert out["name"] == "Follow-up: original query" + assert out["source_count"] == 2 + assert calls == [("rp-spinsingle1", "alice")] + assert session_manager.created is not None + assert session_manager.created.messages + +def test_spinoff_reads_saved_query_for_done_active_task(tmp_path, monkeypatch): + session_id = "rp-activedone1" + data_dir = tmp_path / "deep_research" + _write_research( + data_dir, + session_id, + owner="alice", + result="saved report", + sources=["s1"], + query="completed query", + ) + + class FakeSession: + endpoint_url = "" + model = "" + headers = {} + + def __init__(self): + self.messages = [] + + def add_message(self, message): + self.messages.append(message) + + class FakeSessionManager: + def __init__(self): + self.created = None + + def get_session(self, session_id): + raise KeyError(session_id) + + def create_session(self, **kwargs): + self.created = FakeSession() + return self.created + + def save_sessions(self): + pass + + monkeypatch.setattr( + "routes.research.research_routes.resolve_endpoint", + lambda *_args, **_kwargs: ("http://endpoint/v1", "model", {}), + ) + + handler = _research_handler() + handler._active_tasks[session_id] = {"owner": "alice", "status": "done"} + handler.get_result.return_value = None + handler.get_sources.return_value = [] + + session_manager = FakeSessionManager() + router = setup_research_routes(handler, session_manager=session_manager) + target = _route(router, "/api/research/spinoff/{session_id}", "POST") + + out = asyncio.run(target(session_id=session_id, request=_request("alice"))) + + assert out["name"] == "Follow-up: completed query" + assert out["source_count"] == 1 + assert session_manager.created is not None + primer = session_manager.created.messages[0].content + assert "completed query" in primer + assert "(not recorded)" not in primer From d8d98caa784f4dee791bdfef8ef338abd582b933 Mon Sep 17 00:00:00 2001 From: Ocean Bennett <204957658+undergroundrap@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:21:18 -0400 Subject: [PATCH 26/32] fix(security): sanitize email rich body render path (#5212) --- static/js/document.js | 19 +++++++++++++------ static/js/markdown.js | 3 ++- tests/test_markdown_dom_xss_helpers.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/static/js/document.js b/static/js/document.js index b245ec852..0aadaa503 100644 --- a/static/js/document.js +++ b/static/js/document.js @@ -2378,20 +2378,27 @@ import { bindMenuDismiss, dismissOrRemove } from './escMenuStack.js'; } // ── WYSIWYG email body helpers ── + function _emailPlainTextToHtml(text) { + const d = document.createElement('div'); + d.textContent = text == null ? '' : String(text); + return d.innerHTML.replace(/\n/g, '
'); + } + function _emailBodyToHtml(text) { const t = (text || '').trim(); if (!t) return ''; // If it already contains a formatting/structural HTML tag, it's a saved - // WYSIWYG body — use it verbatim. (Checking a leading '<' isn't enough: a + // WYSIWYG body — sanitize it before rendering. (Checking a leading '<' isn't enough: a // rich body often starts with plain text, e.g. "Hi there".) - if (/<\/?(b|i|u|s|strong|em|del|strike|a|p|div|br|ul|ol|li|h[1-3]|blockquote|span|code|pre)\b[^>]*>/i.test(t)) return t; + if (/<\/?(b|i|u|s|strong|em|del|strike|a|p|div|br|ul|ol|li|h[1-3]|blockquote|span|code|pre)\b[^>]*>/i.test(t)) { + return markdownModule.sanitizeAllowedHtml + ? markdownModule.sanitizeAllowedHtml(t) + : _emailPlainTextToHtml(t); + } // Email body: keep author-typed `:shortcode:` text literal. Issue #345 // (shortcode → emoji) is scoped to chat; do not rewrite colons in mail. try { return markdownModule.mdToHtml(text, { shortcodes: false }); } - catch (_) { - const d = document.createElement('div'); d.textContent = text; - return d.innerHTML.replace(/\n/g, '
'); - } + catch (_) { return _emailPlainTextToHtml(text); } } // Mirror the rich body's plain text into the hidden textarea so the existing // send / draft / change-detection plumbing (which reads the textarea) stays diff --git a/static/js/markdown.js b/static/js/markdown.js index f2bb7f85f..a3b27fbc8 100644 --- a/static/js/markdown.js +++ b/static/js/markdown.js @@ -133,7 +133,7 @@ function _cleanAllowedHtmlOnce(htmlString) { return tpl.innerHTML; } -function sanitizeAllowedHtml(html) { +export function sanitizeAllowedHtml(html) { const raw = String(html == null ? '' : html); // Non-browser context (e.g. a future SSR/Node import): fail closed by // escaping rather than trusting the markup. @@ -824,6 +824,7 @@ export function renderMermaid(container) { const markdownModule = { escapeHtml, mdToHtml, + sanitizeAllowedHtml, squashOutsideCode, renderContent, processWithThinking, diff --git a/tests/test_markdown_dom_xss_helpers.py b/tests/test_markdown_dom_xss_helpers.py index 25b18417d..db9ab9c9b 100644 --- a/tests/test_markdown_dom_xss_helpers.py +++ b/tests/test_markdown_dom_xss_helpers.py @@ -23,3 +23,16 @@ def test_markdown_raw_html_sanitizer_strips_scriptable_css(): assert "if (name === 'style')" in src assert r"javascript:|vbscript:|data:|expression\(" in src assert "el.removeAttribute(attr.name);" in src + + +def test_email_rich_body_render_path_reuses_raw_html_sanitizer(): + markdown_src = (_REPO / "static" / "js" / "markdown.js").read_text(encoding="utf-8") + document_src = (_REPO / "static" / "js" / "document.js").read_text(encoding="utf-8") + email_body_helper = document_src.split("function _emailBodyToHtml(text)", 1)[1].split( + " // Mirror the rich body's plain text", 1 + )[0] + + assert "export function sanitizeAllowedHtml(html)" in markdown_src + assert "sanitizeAllowedHtml," in markdown_src + assert "markdownModule.sanitizeAllowedHtml(t)" in email_body_helper + assert "return t;" not in email_body_helper From a50e30c28bc7cbc24a6ff9e504d92112ae51bf0f Mon Sep 17 00:00:00 2001 From: Harshit Ojha <92013456+harshit-ojha0324@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:35:52 -0400 Subject: [PATCH 27/32] test(integrations): drop redundant trailing-slash assertion The exact-equality assert on the line above (requested_url == WEBHOOK_BASE) already implies the URL has no trailing slash, so the endswith check adds nothing. --- tests/test_integrations_url_join.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_integrations_url_join.py b/tests/test_integrations_url_join.py index 9a6359038..a5e19722e 100644 --- a/tests/test_integrations_url_join.py +++ b/tests/test_integrations_url_join.py @@ -114,4 +114,3 @@ async def test_api_call_root_path_has_no_trailing_slash(): assert result.get("exit_code") == 0 requested_url = mock_client.request.call_args.args[1] assert requested_url == WEBHOOK_BASE - assert not requested_url.endswith("/") From 264da65186dcae20e61e6bec128b75357793a0d4 Mon Sep 17 00:00:00 2001 From: Odysseus Review Oracle Date: Sat, 4 Jul 2026 23:33:43 +0000 Subject: [PATCH 28/32] fix(chat): honor explicit web search denial --- routes/chat_routes.py | 2 -- tests/test_chat_route_tool_policy.py | 31 ++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/routes/chat_routes.py b/routes/chat_routes.py index f7b4cc862..08bab91b9 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -781,11 +781,9 @@ def setup_chat_routes( # by default without having to send allow_bash in every request. if allow_bash is not None and str(allow_bash).lower() != "true": disabled_tools.add("bash") - _explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web") if ( allow_web_search is not None and str(allow_web_search).lower() != "true" - and not _explicit_web_intent ): disabled_tools.add("web_search") disabled_tools.add("web_fetch") diff --git a/tests/test_chat_route_tool_policy.py b/tests/test_chat_route_tool_policy.py index 869b9a972..e43ae525b 100644 --- a/tests/test_chat_route_tool_policy.py +++ b/tests/test_chat_route_tool_policy.py @@ -14,6 +14,8 @@ from pathlib import Path import pytest +from src.action_intents import classify_tool_intent + _CHAT_ROUTES = Path(__file__).resolve().parent.parent / "routes" / "chat_routes.py" @@ -73,7 +75,7 @@ def test_allow_web_search_reads_from_body_as_fallback(): ) -def test_disabled_tools_does_not_bash_when_allow_bash_is_none(): +def test_disabled_tools_respects_missing_vs_explicit_toggles(): """When allow_bash is not set (None), bash must NOT be unconditionally added to disabled_tools. The per-user privilege check handles it. """ @@ -89,8 +91,8 @@ def test_disabled_tools_does_not_bash_when_allow_bash_is_none(): assert "allow_web_search is not None" in source, ( "disabled_tools check must guard against allow_web_search being None" ) - assert "_explicit_web_intent" in source and "not _explicit_web_intent" in source, ( - "explicit web-search requests must override an off web toggle for that turn" + assert "and not _explicit_web_intent" not in source, ( + "explicit allow_web_search=false must not be overridden by prompt web intent" ) @@ -116,7 +118,6 @@ def _build_disabled_tools( if ( allow_web_search is not None and str(allow_web_search).lower() != "true" - and not explicit_web_intent ): disabled_tools.add("web_search") disabled_tools.add("web_fetch") @@ -156,15 +157,27 @@ def test_json_body_allow_web_search_false_disables_web(): assert "web_fetch" in disabled -def test_explicit_web_intent_overrides_false_web_toggle_for_turn(): - """A stale/off web toggle must not remove web tools when the message - explicitly asks to use web search.""" +@pytest.mark.parametrize( + "message", + [ + "please use web search for current CVEs", + "search the web for current CVEs", + "can you look up the latest docs", + ], +) +def test_explicit_false_disables_web_despite_prompt_web_intent(message): + """Explicit allow_web_search=false is a hard deny even when the prompt + asks for web search.""" + intent = classify_tool_intent(message) + assert intent is not None + assert intent.category == "web" + disabled = _build_disabled_tools( allow_web_search="false", explicit_web_intent=True, ) - assert "web_search" not in disabled - assert "web_fetch" not in disabled + assert "web_search" in disabled + assert "web_fetch" in disabled def test_admin_user_gets_bash_enabled_by_default(): From 5acd0ceae9377f070427f38e57c1c74dbb281c6b Mon Sep 17 00:00:00 2001 From: "Tal.Yuan" Date: Sun, 5 Jul 2026 09:58:34 +0800 Subject: [PATCH 29/32] refactor(routes): move contacts domain into routes/contacts/ subpackage (#5227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slice 2e of the route-domain reorganization (#4082/#4071, per specs/architecture-runtime-inventory.md §6.3). Moves contacts_routes.py into routes/contacts/, leaving a backward-compat sys.modules shim at the old path. Pure file reorganization, no behavior change. The shim uses sys.modules replacement (same pattern as the merged gallery #4903, research #4975, memory #5007, and history #5090 slices) so that `import routes.contacts_routes`, `from routes.contacts_routes import X`, `importlib.import_module(...)`, the string-targeted `monkeypatch.setattr("routes.contacts_routes.SETTINGS_FILE", ...)` used by test_carddav_password_encryption.py, and the `import ... as cr` + `setattr(cr, ...)` pattern in test_contacts_add_null_name.py all operate on the same module object the application uses. This also keeps the mutable module state `_contact_cache` identical across import paths. The canonical module does NOT depend on the shim — routes/contacts/ contacts_routes.py imports only from core/, src/, and stdlib (zero internal routes/ coupling). The inbound edge from routes/email_helpers.py (imports _fetch_contacts) keeps working through the shim. Zero source-introspection landmines — no test reads this file by path. Adds tests/test_contacts_routes_shim.py to pin the sys.modules shim contract (same-object + string-targeted monkeypatch reach-through). Verified: compileall clean; full suite 4485 passed, 3 skipped. --- app.py | 2 +- routes/contacts/__init__.py | 5 + routes/contacts/contacts_routes.py | 900 ++++++++++++++++++++++++++++ routes/contacts_routes.py | 910 +---------------------------- tests/test_contacts_routes_shim.py | 46 ++ 5 files changed, 967 insertions(+), 896 deletions(-) create mode 100644 routes/contacts/__init__.py create mode 100644 routes/contacts/contacts_routes.py create mode 100644 tests/test_contacts_routes_shim.py diff --git a/app.py b/app.py index 8914893ab..f2f1dd0b3 100644 --- a/app.py +++ b/app.py @@ -813,7 +813,7 @@ from routes.vault_routes import setup_vault_routes app.include_router(setup_vault_routes()) # Contacts (CardDAV) -from routes.contacts_routes import setup_contacts_routes +from routes.contacts.contacts_routes import setup_contacts_routes app.include_router(setup_contacts_routes()) from companion import setup_companion_routes diff --git a/routes/contacts/__init__.py b/routes/contacts/__init__.py new file mode 100644 index 000000000..382f8f848 --- /dev/null +++ b/routes/contacts/__init__.py @@ -0,0 +1,5 @@ +"""Contacts route domain package (slice 2e, #4082/#4071). + +Contains contacts_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/contacts_routes.py re-exports from here. +""" diff --git a/routes/contacts/contacts_routes.py b/routes/contacts/contacts_routes.py new file mode 100644 index 000000000..81024ed6c --- /dev/null +++ b/routes/contacts/contacts_routes.py @@ -0,0 +1,900 @@ +""" +contacts_routes.py + +CardDAV contacts integration. Reads from local Radicale, supports +search and adding new contacts. +""" + +import re +import logging +import uuid +import json +import csv +import io +import os +import inspect +import httpx +from pathlib import Path +from datetime import datetime +from urllib.parse import urljoin, urlparse, urlunparse + +from core.log_safety import redact_url +from fastapi import APIRouter, Query, Depends, Response, HTTPException +from typing import List, Dict, Optional + +from core.middleware import require_admin +from src.url_safety import check_outbound_url + +logger = logging.getLogger(__name__) + +from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE +DATA_DIR = Path(_DATA_DIR) +SETTINGS_FILE = Path(_SETTINGS_FILE) +LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE) + + +def _load_settings(): + if SETTINGS_FILE.exists(): + return json.loads(SETTINGS_FILE.read_text(encoding="utf-8")) + return {} + + +def _save_settings(settings): + from core.atomic_io import atomic_write_json + atomic_write_json(str(SETTINGS_FILE), settings, indent=2) + + +def _get_carddav_config(): + import os + settings = _load_settings() + password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")) + if password and "carddav_password" in settings: + from src.secret_storage import decrypt + password = decrypt(password) + return { + "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")), + "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")), + "password": password, + } + + +def _carddav_configured(cfg: Optional[Dict] = None) -> bool: + cfg = cfg or _get_carddav_config() + return bool((cfg.get("url") or "").strip()) + + +def _validate_carddav_url(url: str) -> str: + cleaned = (url if isinstance(url, str) else "").strip().rstrip("/") + ok, reason = check_outbound_url( + cleaned, + block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise ValueError(f"Rejected CardDAV URL: {reason}") + return cleaned + + +def _carddav_base_url(cfg: Dict) -> str: + return _validate_carddav_url(cfg.get("url") or "") + + +def _normalize_contact(contact: Dict) -> Dict: + emails = [] + for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]): + e = str(e or "").strip() + if e and e not in emails: + emails.append(e) + phones = [] + for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]): + p = str(p or "").strip() + if p and p not in phones: + phones.append(p) + name = str(contact.get("name") or "").strip() + if not name and emails: + name = emails[0].split("@")[0] + address = str(contact.get("address") or "").strip() + return { + "uid": str(contact.get("uid") or uuid.uuid4()), + "name": name, + "emails": emails, + "phones": phones, + "address": address, + } + + +def _load_local_contacts() -> List[Dict]: + try: + if not LOCAL_CONTACTS_FILE.exists(): + return [] + data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8")) + rows = data.get("contacts", data) if isinstance(data, dict) else data + return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)] + except Exception as e: + logger.error(f"Failed to load local contacts: {e}") + return [] + + +def _save_local_contacts(contacts: List[Dict]) -> None: + from core.atomic_io import atomic_write_json + DATA_DIR.mkdir(parents=True, exist_ok=True) + atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2) + _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts] + _contact_cache["fetched_at"] = datetime.utcnow() + + +# ── vCard parsing ── + +def _vunesc(value: str) -> str: + """Reverse _vesc() — turn escaped vCard text back into the raw value. + Order matters: handle \\n/\\, /\\; first, backslash-unescape last.""" + if not value: + return value + out = [] + i = 0 + while i < len(value): + ch = value[i] + if ch == "\\" and i + 1 < len(value): + nxt = value[i + 1] + if nxt in ("n", "N"): + out.append("\n") + elif nxt in (",", ";", "\\"): + out.append(nxt) + else: + out.append(nxt) + i += 2 + else: + out.append(ch) + i += 1 + return "".join(out) + + +def _parse_vcards(text: str) -> List[Dict]: + """Parse a stream of vCards into dicts with name, email, phone.""" + # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single + # space or tab is a continuation of the previous logical line. Real + # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN / + # PHOTO lines, and splitting on raw newlines without unfolding dropped the + # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the + # email/name. + text = re.sub(r"\r\n[ \t]", "", text or "") + text = re.sub(r"\n[ \t]", "", text) + contacts = [] + for block in re.split(r"BEGIN:VCARD", text): + if not block.strip(): + continue + contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""} + for line in block.split("\n"): + line = line.strip() + # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...") + # that Apple Contacts / iCloud / many CardDAV servers emit by + # default — without this the property-name checks below miss those + # lines and silently drop the email / phone. The group token only + # precedes the property name, so it is safe to strip for matching + # and value extraction, and a no-op for non-grouped lines. + name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1) + if name_part.startswith("FN:") or name_part.startswith("FN;"): + contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else "" + elif name_part.startswith("EMAIL"): + # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar + if ":" in name_part: + email_addr = _vunesc(name_part.split(":", 1)[1]) + if email_addr and email_addr not in contact["emails"]: + contact["emails"].append(email_addr) + elif name_part.startswith("TEL"): + if ":" in name_part: + phone = _vunesc(name_part.split(":", 1)[1]) + if phone and phone not in contact["phones"]: + contact["phones"].append(phone) + elif name_part.startswith("ADR"): + # vCard ADR is 7 semicolon-separated components: + # post-office-box;extended-address;street;locality;region;postal-code;country. + # Recover a human-readable string by joining non-empty + # components with ", ". + if ":" in name_part: + raw = name_part.split(":", 1)[1] + parts = [_vunesc(p).strip() for p in raw.split(";")] + contact["address"] = ", ".join(p for p in parts if p) + elif name_part.startswith("UID:"): + contact["uid"] = _vunesc(name_part[4:]) + if contact["name"] or contact["emails"]: + contacts.append(contact) + return contacts + + +def _vesc(value: str) -> str: + """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma, + semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd' + or any value containing a newline produces a malformed vCard (broken + N/FN fields) or could inject arbitrary properties.""" + return ( + (value or "") + .replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\r", "") + .replace(",", "\\,") + .replace(";", "\\;") + ) + + +def _build_vcard(name: str, email: str, uid: Optional[str] = None, + emails: Optional[List[str]] = None, + phones: Optional[List[str]] = None, + address: Optional[str] = None) -> str: + """Build a vCard. Accepts either a single `email` (legacy callers) or + full `emails`/`phones` lists (edit path). The first email is marked + PREF=1. All values are RFC-6350-escaped.""" + if not uid: + uid = str(uuid.uuid4()) + # Normalize email lists — `email` arg is a convenience for single-email + # creation; `emails` (if given) is authoritative. + email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()] + phone_list = [p.strip() for p in (phones or []) if p and p.strip()] + # Try to split name into first/last + parts = name.strip().split() + if len(parts) >= 2: + first = parts[0] + last = " ".join(parts[1:]) + else: + first = name + last = "" + # N field is structured (5 components separated by ';') — escape each + # component individually so a comma in the name doesn't split it. + n_field = f"{_vesc(last)};{_vesc(first)};;;" + lines = [ + "BEGIN:VCARD", + "VERSION:4.0", + f"UID:{_vesc(uid)}", + f"FN:{_vesc(name)}", + f"N:{n_field}", + ] + for i, em in enumerate(email_list): + # First email is the preferred one. + lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}") + for ph in phone_list: + lines.append(f"TEL:{_vesc(ph)}") + # Address: stuff the whole human-readable string into the street + # component of ADR. vCard ADR has 7 semicolon-separated components: + # post-office-box;extended-address;street;locality;region;postal-code;country. + addr = (address or "").strip() + if addr: + lines.append(f"ADR:;;{_vesc(addr)};;;;") + lines.append("END:VCARD") + return "\r\n".join(lines) + "\r\n" + + +# ── In-memory cache ── + +_contact_cache = {"contacts": [], "fetched_at": None} + + +def _abs_url(href: str) -> str: + """Combine a multistatus (an absolute path like + /user/contacts/x.vcf) with the configured CardDAV server origin so we + get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only + for the configured origin; a cross-origin href is treated as a path on the + configured server so a malicious CardDAV response cannot redirect later + writes/deletes to cloud metadata or another host.""" + cfg = _get_carddav_config() + base = _carddav_base_url(cfg) + base_p = urlparse(base) + joined = urljoin(base.rstrip("/") + "/", href or "") + joined_p = urlparse(joined) + if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc): + joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, "")) + return _validate_carddav_url(joined) + + +# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request, +# alongside the resource href. Lets us map each contact's UID to the real +# server resource path (which is NOT always .vcf for contacts created +# by other clients). +_ADDRESSBOOK_QUERY = ( + '' + '' + '' + '' + '' +) + + +def _fetch_via_report(cfg, auth): + """Try a CardDAV REPORT addressbook-query — returns contacts WITH an + `href` field, or None if the server doesn't support it / errors.""" + from defusedxml import ElementTree as ET + try: + r = httpx.request( + "REPORT", cfg["url"], + content=_ADDRESSBOOK_QUERY.encode("utf-8"), + headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"}, + auth=auth, timeout=10, + ) + if r.status_code not in (207, 200): + return None + root = ET.fromstring(r.text) + ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"} + out = [] + for resp in root.findall("D:response", ns): + href_el = resp.find("D:href", ns) + data_el = resp.find(".//C:address-data", ns) + if href_el is None or data_el is None or not (data_el.text or "").strip(): + continue + parsed = _parse_vcards(data_el.text) + if not parsed: + continue + c = parsed[0] + c["href"] = href_el.text.strip() + out.append(c) + # If the REPORT parsed to ZERO contacts, don't trust it — some + # CardDAV servers treat an empty as "match nothing" and + # return a valid-but-empty 207. Return None so the caller falls + # back to the plain GET (which lists everything). A genuinely empty + # address book just costs one extra GET that also returns nothing. + if not out: + return None + return out + except Exception as e: + logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}") + return None + + +def _fetch_contacts(force=False): + """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON.""" + if not force and _contact_cache["fetched_at"]: + age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds() + if age < 60: + return _contact_cache["contacts"] + + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + _contact_cache["contacts"] = contacts + _contact_cache["fetched_at"] = datetime.utcnow() + return contacts + + try: + cfg["url"] = _carddav_base_url(cfg) + auth = None + if cfg["username"]: + auth = (cfg["username"], cfg["password"]) + # Preferred path: REPORT gives us hrefs for reliable edit/delete. + contacts = _fetch_via_report(cfg, auth) + if contacts is None: + # Fallback: plain GET, concatenated vCards, no hrefs. + r = httpx.get(cfg["url"], auth=auth, timeout=10) + if r.status_code != 200: + logger.warning(f"CardDAV returned {r.status_code}") + return _contact_cache["contacts"] + contacts = _parse_vcards(r.text) + _contact_cache["contacts"] = contacts + _contact_cache["fetched_at"] = datetime.utcnow() + return contacts + except Exception as e: + logger.error(f"Failed to fetch contacts: {e}") + return _contact_cache["contacts"] + + +def _resolve_resource_url(uid: str) -> str: + """Map a contact UID to its real CardDAV resource URL. Uses the href + captured during fetch when available (handles contacts whose filename + != UID); falls back to the .vcf guess for app-created contacts or + when no href is known.""" + def _lookup(): + for c in _contact_cache.get("contacts", []): + if c.get("uid") == uid and c.get("href"): + return _abs_url(c["href"]) + return None + found = _lookup() + if found: + return found + # Not in cache (or no href) — refresh once and retry before guessing. + try: + _fetch_contacts(force=True) + except Exception: + pass + return _lookup() or _vcard_url(uid) + + +def _create_contact(name: str, email: str, address: str = "") -> bool: + """Add a new contact via CardDAV or local contacts.""" + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + email_l = (email or "").strip().lower() + for c in contacts: + if email_l and email_l in [e.lower() for e in c.get("emails", [])]: + return True + contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address})) + _save_local_contacts(contacts) + return True + + contact_uid = str(uuid.uuid4()) + vcard = _build_vcard(name, email, contact_uid, address=address) + try: + url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf" + auth = None + if cfg["username"]: + auth = (cfg["username"], cfg["password"]) + r = httpx.put( + url, + data=vcard.encode("utf-8"), + headers={"Content-Type": "text/vcard; charset=utf-8"}, + auth=auth, + timeout=10, + ) + if r.status_code in (200, 201, 204): + # Invalidate cache + _contact_cache["fetched_at"] = None + return True + logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}") + return False + except Exception as e: + logger.error(f"Failed to create contact: {e}") + return False + + +def _vcard_url(uid: str) -> str: + """The CardDAV resource URL for a given contact UID. The uid is URL- + encoded so a value containing '/', '..' or other path chars can't + escape the collection and target an arbitrary CardDAV resource.""" + from urllib.parse import quote + cfg = _get_carddav_config() + return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf" + + +def _import_vcards(text: str) -> Dict: + """Import a (possibly multi-card) .vcf blob. Each card is PUT to the + CardDAV server PRESERVING its full original content (ADR/ORG/photo/ + etc.) — we don't rebuild it, just ensure it has VERSION + UID and + normalize line endings. Returns {imported, failed, total}.""" + from urllib.parse import quote + cfg = _get_carddav_config() + if not cfg.get("url"): + parsed = _parse_vcards(text) + contacts = _load_local_contacts() + existing = { + e.lower() + for c in contacts + for e in (c.get("emails") or []) + if e + } + imported = 0 + for c in parsed: + emails = [e for e in (c.get("emails") or []) if e] + if emails and any(e.lower() in existing for e in emails): + continue + contacts.append(_normalize_contact(c)) + for e in emails: + existing.add(e.lower()) + imported += 1 + if imported: + _save_local_contacts(contacts) + return {"imported": imported, "failed": 0, "total": len(parsed)} + try: + base_url = _carddav_base_url(cfg) + except ValueError as e: + logger.warning("CardDAV import URL rejected: %s", e) + return {"imported": 0, "failed": 0, "total": 0, "error": str(e)} + auth = (cfg["username"], cfg["password"]) if cfg["username"] else None + # Split into individual cards. re.split drops the BEGIN line, so we + # re-add it. Normalize CRLF. + raw = (text or "").replace("\r\n", "\n").replace("\r", "\n") + blocks = [] + for chunk in raw.split("BEGIN:VCARD"): + chunk = chunk.strip() + if not chunk: + continue + # Trim anything after END:VCARD (defensive). + end = chunk.upper().find("END:VCARD") + body = chunk[: end + len("END:VCARD")] if end != -1 else chunk + blocks.append("BEGIN:VCARD\n" + body) + imported = 0 + failed = 0 + for block in blocks: + # Extract or assign a UID. + m = re.search(r"^UID:(.+)$", block, re.MULTILINE) + uid = (m.group(1).strip() if m else "") or str(uuid.uuid4()) + if not m: + # Inject a UID right after the VERSION line (or after BEGIN). + if re.search(r"^VERSION:", block, re.MULTILINE): + block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE) + else: + block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1) + elif not re.search(r"^VERSION:", block, re.MULTILINE): + block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1) + vcard = block.replace("\n", "\r\n") + "\r\n" + url = base_url + "/" + quote(uid, safe="") + ".vcf" + try: + r = httpx.put( + url, data=vcard.encode("utf-8"), + headers={"Content-Type": "text/vcard; charset=utf-8"}, + auth=auth, timeout=15, + ) + if r.status_code in (200, 201, 204): + imported += 1 + else: + failed += 1 + logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}") + except Exception as e: + failed += 1 + logger.error(f"Import PUT {uid} failed: {e}") + if imported: + _contact_cache["fetched_at"] = None + return {"imported": imported, "failed": failed, "total": len(blocks)} + + +def _import_csv_contacts(text: str) -> Dict: + """Import contacts from CSV. Supports common headers: + name/full_name/display_name, email/email_address/e-mail, phone/tel. + Falls back to first columns as name,email,phone when no headers exist.""" + raw = (text or "").strip() + if not raw: + return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"} + + try: + sample = raw[:2048] + dialect = csv.Sniffer().sniff(sample) + except Exception: + dialect = csv.excel + + stream = io.StringIO(raw) + try: + has_header = csv.Sniffer().has_header(raw[:2048]) + except Exception: + has_header = True + + rows = [] + if has_header: + reader = csv.DictReader(stream, dialect=dialect) + for row in reader: + lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()} + name = ( + lowered.get("name") or lowered.get("full name") or lowered.get("full_name") + or lowered.get("display name") or lowered.get("display_name") + or lowered.get("fn") or "" + ) + email = ( + lowered.get("email") or lowered.get("email address") + or lowered.get("email_address") or lowered.get("e-mail") + or lowered.get("mail") or "" + ) + phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or "" + rows.append((name, email, phone)) + else: + stream.seek(0) + reader = csv.reader(stream, dialect=dialect) + for row in reader: + cols = [(c or "").strip() for c in row] + if not any(cols): + continue + rows.append(( + cols[0] if len(cols) > 0 else "", + cols[1] if len(cols) > 1 else "", + cols[2] if len(cols) > 2 else "", + )) + + imported = 0 + failed = 0 + total = 0 + existing_emails = { + e.lower() + for c in _fetch_contacts() + for e in (c.get("emails") or []) + if e + } + for name, email, phone in rows: + email = (email or "").strip() + name = (name or "").strip() or (email.split("@")[0] if email else "") + if not email: + continue + total += 1 + if email.lower() in existing_emails: + continue + ok = _create_contact(name, email) + if ok: + imported += 1 + existing_emails.add(email.lower()) + # If the CSV had a phone number, rewrite the just-created row + # through the richer update path so phone lands in CardDAV too. + if phone: + try: + contacts = _fetch_contacts(force=True) + created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None) + if created and created.get("uid"): + _update_contact(created["uid"], name, [email], [phone]) + except Exception: + pass + else: + failed += 1 + + if imported: + _contact_cache["fetched_at"] = None + return {"imported": imported, "failed": failed, "total": total} + + +def _contacts_to_vcf(contacts: List[Dict]) -> str: + return "".join( + _build_vcard( + c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"), + "", + uid=c.get("uid") or str(uuid.uuid4()), + emails=c.get("emails") or [], + phones=c.get("phones") or [], + ) + for c in contacts + ) + + +def _contacts_to_csv(contacts: List[Dict]) -> str: + out = io.StringIO() + writer = csv.writer(out) + writer.writerow(["name", "email", "phone"]) + for c in contacts: + emails = c.get("emails") or [""] + phones = c.get("phones") or [""] + max_len = max(len(emails), len(phones), 1) + for i in range(max_len): + writer.writerow([ + c.get("name") or "", + emails[i] if i < len(emails) else "", + phones[i] if i < len(phones) else "", + ]) + return out.getvalue() + + +def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool: + """Rewrite an existing contact via CardDAV or local contacts.""" + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + found = False + out = [] + for c in contacts: + if c.get("uid") == uid: + # Preserve existing address when caller passes "" (only + # updating name/emails/phones, not touching address). + addr = address if address else c.get("address", "") + out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr})) + found = True + else: + out.append(c) + if not found: + out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address})) + _save_local_contacts(out) + return True + + vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address) + # Use the real resource href (handles externally-created contacts whose + # filename != UID); falls back to the .vcf guess. + try: + url = _resolve_resource_url(uid) + auth = (cfg["username"], cfg["password"]) if cfg["username"] else None + r = httpx.put( + url, + data=vcard.encode("utf-8"), + headers={"Content-Type": "text/vcard; charset=utf-8"}, + auth=auth, + timeout=10, + ) + if r.status_code in (200, 201, 204): + _contact_cache["fetched_at"] = None + return True + logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}") + return False + except Exception as e: + logger.error(f"Failed to update contact: {e}") + return False + + +def _delete_contact(uid: str) -> bool: + """Delete a contact via CardDAV or local contacts.""" + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + remaining = [c for c in contacts if c.get("uid") != uid] + _save_local_contacts(remaining) + return True + + try: + url = _resolve_resource_url(uid) + auth = (cfg["username"], cfg["password"]) if cfg["username"] else None + r = httpx.delete(url, auth=auth, timeout=10) + if r.status_code in (200, 204, 404): + # Invalidate cache so the next fetch sees the server truth. + _contact_cache["fetched_at"] = None + # Verify: force a fresh fetch and check the UID is actually gone. + # A 404 on the guessed URL ({uid}.vcf) can mean the contact + # lives at a different resource URL — the DELETE missed it but + # we'd silently report success. This check catches that. + fresh = _fetch_contacts(force=True) + still_there = any(c.get("uid") == uid for c in fresh) + if still_there: + logger.warning( + f"CardDAV DELETE reported success for {uid} " + f"but UID still present after re-fetch — " + f"resource URL may differ from {redact_url(url)}" + ) + return False + if r.status_code == 404: + logger.info(f"CardDAV DELETE 404 for {uid} — already gone") + return True + logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}") + return False + except Exception as e: + logger.error(f"Failed to delete contact: {e}") + return False + + +# ── Routes ── + +def setup_contacts_routes(): + router = APIRouter(prefix="/api/contacts", tags=["contacts"]) + + @router.get("/list") + async def list_contacts(_admin: str = Depends(require_admin)): + """List all contacts.""" + contacts = _fetch_contacts() + return {"contacts": contacts, "count": len(contacts)} + + @router.get("/search") + async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)): + """Search contacts by name or email. Returns up to 10 matches.""" + contacts = _fetch_contacts() + if not q: + return {"results": []} + q_lower = q.lower() + results = [] + for c in contacts: + if q_lower in c["name"].lower(): + results.append(c) + continue + for em in c["emails"]: + if q_lower in em.lower(): + results.append(c) + break + return {"results": results[:10]} + + @router.post("/add") + async def add_contact(data: dict, _admin: str = Depends(require_admin)): + """Add a new contact.""" + name = (data.get("name") or "").strip() + email = (data.get("email") or "").strip() + phone = (data.get("phone") or "").strip() + address = (data.get("address") or "").strip() + if not email: + return {"success": False, "error": "Email required"} + # Check if already exists by email + if email: + contacts = _fetch_contacts() + for c in contacts: + if email.lower() in [e.lower() for e in c["emails"]]: + return {"success": True, "message": "Already exists", "contact": c} + if not name: + name = email.split("@")[0] + create_params = inspect.signature(_create_contact).parameters + if len(create_params) >= 3: + ok = _create_contact(name, email, address) + else: + ok = _create_contact(name, email) + # If a phone was provided, do an immediate update to thread it + # through (the simple _create_contact signature only takes name + + # email + address; phones happen via update). + if ok and phone: + try: + fresh = _fetch_contacts(force=True) + created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None) + if created: + _update_contact( + created["uid"], name, + created.get("emails", []), + [phone], + address, + ) + except Exception: + pass + return {"success": ok} + + @router.post("/import") + async def import_vcf(data: dict, _admin: str = Depends(require_admin)): + """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}.""" + # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list + # in the JSON body) would otherwise reach .strip() and 500 with an + # AttributeError instead of degrading to a clean "no data" response. + text = str(data.get("vcf") or data.get("text") or "") + csv_text = str(data.get("csv") or "") + if text.strip(): + if "BEGIN:VCARD" not in text.upper(): + return {"success": False, "error": "No vCard data found"} + result = _import_vcards(text) + elif csv_text.strip(): + result = _import_csv_contacts(csv_text) + else: + return {"success": False, "error": "No contact data found"} + result["success"] = result.get("imported", 0) > 0 + return result + + @router.get("/export") + async def export_contacts( + format: str = Query("vcf", pattern="^(vcf|csv)$"), + _admin: str = Depends(require_admin), + ): + """Export all contacts as vCard or CSV.""" + contacts = _fetch_contacts(force=True) + if format == "csv": + content = _contacts_to_csv(contacts) + media_type = "text/csv; charset=utf-8" + filename = "odysseus-contacts.csv" + else: + content = _contacts_to_vcf(contacts) + media_type = "text/vcard; charset=utf-8" + filename = "odysseus-contacts.vcf" + return Response( + content=content, + media_type=media_type, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + @router.get("/config") + async def get_config(_admin: str = Depends(require_admin)): + cfg = _get_carddav_config() + # Mask password + if cfg["password"]: + cfg["password"] = "***" + return cfg + + @router.put("/config") + async def update_config(data: dict, _admin: str = Depends(require_admin)): + settings = _load_settings() + for key in ("carddav_url", "carddav_username", "carddav_password"): + if key in data: + if key == "carddav_url" and str(data[key] or "").strip(): + try: + settings[key] = _validate_carddav_url(data[key]) + except ValueError as e: + raise HTTPException(400, str(e)) + else: + value = data[key] + if key == "carddav_password" and value: + from src.secret_storage import encrypt + value = encrypt(value) + settings[key] = value + _save_settings(settings) + # Force re-fetch + _contact_cache["fetched_at"] = None + return {"success": True} + + @router.delete("/clear") + async def clear_contacts(_admin: str = Depends(require_admin)): + """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache.""" + _save_local_contacts([]) + return {"success": True} + + # NOTE: the /{uid} routes are declared LAST so the literal paths above + # (/list, /search, /add, /config) win — otherwise PUT /config would + # match PUT /{uid} with uid="config". + @router.put("/{uid}") + async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)): + """Edit an existing contact — name / emails / phones / address.""" + name = (data.get("name") or "").strip() + emails = data.get("emails") + phones = data.get("phones") + if emails is None and data.get("email"): + emails = [data["email"]] + emails = [e.strip() for e in (emails or []) if e and e.strip()] + phones = [p.strip() for p in (phones or []) if p and p.strip()] + address = (data.get("address") or "").strip() + if not name and not emails and not address: + return {"success": False, "error": "Name, email, or address required"} + if not name and emails: + name = emails[0].split("@")[0] + ok = _update_contact(uid, name, emails, phones, address) + return {"success": ok} + + @router.delete("/{uid}") + async def delete_contact(uid: str, _admin: str = Depends(require_admin)): + """Delete a contact by UID.""" + if not uid: + return {"success": False, "error": "UID required"} + ok = _delete_contact(uid) + return {"success": ok} + + return router diff --git a/routes/contacts_routes.py b/routes/contacts_routes.py index 81024ed6c..eb0cca7a9 100644 --- a/routes/contacts_routes.py +++ b/routes/contacts_routes.py @@ -1,900 +1,20 @@ -""" -contacts_routes.py +"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py. -CardDAV contacts integration. Reads from local Radicale, supports -search and adding new contacts. +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``, +``importlib.import_module("routes.contacts_routes")``, and the string-targeted +``monkeypatch.setattr("routes.contacts_routes.SETTINGS_FILE", ...)`` pattern +used by test_carddav_password_encryption.py / test_contacts_carddav_security.py +— plus the ``import ... as contacts_routes`` + ``setattr(...)`` pattern in +test_contacts_add_null_name.py — all operate on the *same* object the +application actually uses. This also keeps ``_contact_cache`` (mutable module +state) identical across import paths. Keeps existing import paths working +after slice 2e (#4082/#4071). No source-introspection tests read this file +by path. """ -import re -import logging -import uuid -import json -import csv -import io -import os -import inspect -import httpx -from pathlib import Path -from datetime import datetime -from urllib.parse import urljoin, urlparse, urlunparse +import sys as _sys -from core.log_safety import redact_url -from fastapi import APIRouter, Query, Depends, Response, HTTPException -from typing import List, Dict, Optional +from routes.contacts import contacts_routes as _canonical # noqa: F401 -from core.middleware import require_admin -from src.url_safety import check_outbound_url - -logger = logging.getLogger(__name__) - -from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE -DATA_DIR = Path(_DATA_DIR) -SETTINGS_FILE = Path(_SETTINGS_FILE) -LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE) - - -def _load_settings(): - if SETTINGS_FILE.exists(): - return json.loads(SETTINGS_FILE.read_text(encoding="utf-8")) - return {} - - -def _save_settings(settings): - from core.atomic_io import atomic_write_json - atomic_write_json(str(SETTINGS_FILE), settings, indent=2) - - -def _get_carddav_config(): - import os - settings = _load_settings() - password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")) - if password and "carddav_password" in settings: - from src.secret_storage import decrypt - password = decrypt(password) - return { - "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")), - "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")), - "password": password, - } - - -def _carddav_configured(cfg: Optional[Dict] = None) -> bool: - cfg = cfg or _get_carddav_config() - return bool((cfg.get("url") or "").strip()) - - -def _validate_carddav_url(url: str) -> str: - cleaned = (url if isinstance(url, str) else "").strip().rstrip("/") - ok, reason = check_outbound_url( - cleaned, - block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true", - ) - if not ok: - raise ValueError(f"Rejected CardDAV URL: {reason}") - return cleaned - - -def _carddav_base_url(cfg: Dict) -> str: - return _validate_carddav_url(cfg.get("url") or "") - - -def _normalize_contact(contact: Dict) -> Dict: - emails = [] - for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]): - e = str(e or "").strip() - if e and e not in emails: - emails.append(e) - phones = [] - for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]): - p = str(p or "").strip() - if p and p not in phones: - phones.append(p) - name = str(contact.get("name") or "").strip() - if not name and emails: - name = emails[0].split("@")[0] - address = str(contact.get("address") or "").strip() - return { - "uid": str(contact.get("uid") or uuid.uuid4()), - "name": name, - "emails": emails, - "phones": phones, - "address": address, - } - - -def _load_local_contacts() -> List[Dict]: - try: - if not LOCAL_CONTACTS_FILE.exists(): - return [] - data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8")) - rows = data.get("contacts", data) if isinstance(data, dict) else data - return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)] - except Exception as e: - logger.error(f"Failed to load local contacts: {e}") - return [] - - -def _save_local_contacts(contacts: List[Dict]) -> None: - from core.atomic_io import atomic_write_json - DATA_DIR.mkdir(parents=True, exist_ok=True) - atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2) - _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts] - _contact_cache["fetched_at"] = datetime.utcnow() - - -# ── vCard parsing ── - -def _vunesc(value: str) -> str: - """Reverse _vesc() — turn escaped vCard text back into the raw value. - Order matters: handle \\n/\\, /\\; first, backslash-unescape last.""" - if not value: - return value - out = [] - i = 0 - while i < len(value): - ch = value[i] - if ch == "\\" and i + 1 < len(value): - nxt = value[i + 1] - if nxt in ("n", "N"): - out.append("\n") - elif nxt in (",", ";", "\\"): - out.append(nxt) - else: - out.append(nxt) - i += 2 - else: - out.append(ch) - i += 1 - return "".join(out) - - -def _parse_vcards(text: str) -> List[Dict]: - """Parse a stream of vCards into dicts with name, email, phone.""" - # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single - # space or tab is a continuation of the previous logical line. Real - # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN / - # PHOTO lines, and splitting on raw newlines without unfolding dropped the - # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the - # email/name. - text = re.sub(r"\r\n[ \t]", "", text or "") - text = re.sub(r"\n[ \t]", "", text) - contacts = [] - for block in re.split(r"BEGIN:VCARD", text): - if not block.strip(): - continue - contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""} - for line in block.split("\n"): - line = line.strip() - # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...") - # that Apple Contacts / iCloud / many CardDAV servers emit by - # default — without this the property-name checks below miss those - # lines and silently drop the email / phone. The group token only - # precedes the property name, so it is safe to strip for matching - # and value extraction, and a no-op for non-grouped lines. - name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1) - if name_part.startswith("FN:") or name_part.startswith("FN;"): - contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else "" - elif name_part.startswith("EMAIL"): - # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar - if ":" in name_part: - email_addr = _vunesc(name_part.split(":", 1)[1]) - if email_addr and email_addr not in contact["emails"]: - contact["emails"].append(email_addr) - elif name_part.startswith("TEL"): - if ":" in name_part: - phone = _vunesc(name_part.split(":", 1)[1]) - if phone and phone not in contact["phones"]: - contact["phones"].append(phone) - elif name_part.startswith("ADR"): - # vCard ADR is 7 semicolon-separated components: - # post-office-box;extended-address;street;locality;region;postal-code;country. - # Recover a human-readable string by joining non-empty - # components with ", ". - if ":" in name_part: - raw = name_part.split(":", 1)[1] - parts = [_vunesc(p).strip() for p in raw.split(";")] - contact["address"] = ", ".join(p for p in parts if p) - elif name_part.startswith("UID:"): - contact["uid"] = _vunesc(name_part[4:]) - if contact["name"] or contact["emails"]: - contacts.append(contact) - return contacts - - -def _vesc(value: str) -> str: - """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma, - semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd' - or any value containing a newline produces a malformed vCard (broken - N/FN fields) or could inject arbitrary properties.""" - return ( - (value or "") - .replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "") - .replace(",", "\\,") - .replace(";", "\\;") - ) - - -def _build_vcard(name: str, email: str, uid: Optional[str] = None, - emails: Optional[List[str]] = None, - phones: Optional[List[str]] = None, - address: Optional[str] = None) -> str: - """Build a vCard. Accepts either a single `email` (legacy callers) or - full `emails`/`phones` lists (edit path). The first email is marked - PREF=1. All values are RFC-6350-escaped.""" - if not uid: - uid = str(uuid.uuid4()) - # Normalize email lists — `email` arg is a convenience for single-email - # creation; `emails` (if given) is authoritative. - email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()] - phone_list = [p.strip() for p in (phones or []) if p and p.strip()] - # Try to split name into first/last - parts = name.strip().split() - if len(parts) >= 2: - first = parts[0] - last = " ".join(parts[1:]) - else: - first = name - last = "" - # N field is structured (5 components separated by ';') — escape each - # component individually so a comma in the name doesn't split it. - n_field = f"{_vesc(last)};{_vesc(first)};;;" - lines = [ - "BEGIN:VCARD", - "VERSION:4.0", - f"UID:{_vesc(uid)}", - f"FN:{_vesc(name)}", - f"N:{n_field}", - ] - for i, em in enumerate(email_list): - # First email is the preferred one. - lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}") - for ph in phone_list: - lines.append(f"TEL:{_vesc(ph)}") - # Address: stuff the whole human-readable string into the street - # component of ADR. vCard ADR has 7 semicolon-separated components: - # post-office-box;extended-address;street;locality;region;postal-code;country. - addr = (address or "").strip() - if addr: - lines.append(f"ADR:;;{_vesc(addr)};;;;") - lines.append("END:VCARD") - return "\r\n".join(lines) + "\r\n" - - -# ── In-memory cache ── - -_contact_cache = {"contacts": [], "fetched_at": None} - - -def _abs_url(href: str) -> str: - """Combine a multistatus (an absolute path like - /user/contacts/x.vcf) with the configured CardDAV server origin so we - get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only - for the configured origin; a cross-origin href is treated as a path on the - configured server so a malicious CardDAV response cannot redirect later - writes/deletes to cloud metadata or another host.""" - cfg = _get_carddav_config() - base = _carddav_base_url(cfg) - base_p = urlparse(base) - joined = urljoin(base.rstrip("/") + "/", href or "") - joined_p = urlparse(joined) - if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc): - joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, "")) - return _validate_carddav_url(joined) - - -# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request, -# alongside the resource href. Lets us map each contact's UID to the real -# server resource path (which is NOT always .vcf for contacts created -# by other clients). -_ADDRESSBOOK_QUERY = ( - '' - '' - '' - '' - '' -) - - -def _fetch_via_report(cfg, auth): - """Try a CardDAV REPORT addressbook-query — returns contacts WITH an - `href` field, or None if the server doesn't support it / errors.""" - from defusedxml import ElementTree as ET - try: - r = httpx.request( - "REPORT", cfg["url"], - content=_ADDRESSBOOK_QUERY.encode("utf-8"), - headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"}, - auth=auth, timeout=10, - ) - if r.status_code not in (207, 200): - return None - root = ET.fromstring(r.text) - ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"} - out = [] - for resp in root.findall("D:response", ns): - href_el = resp.find("D:href", ns) - data_el = resp.find(".//C:address-data", ns) - if href_el is None or data_el is None or not (data_el.text or "").strip(): - continue - parsed = _parse_vcards(data_el.text) - if not parsed: - continue - c = parsed[0] - c["href"] = href_el.text.strip() - out.append(c) - # If the REPORT parsed to ZERO contacts, don't trust it — some - # CardDAV servers treat an empty as "match nothing" and - # return a valid-but-empty 207. Return None so the caller falls - # back to the plain GET (which lists everything). A genuinely empty - # address book just costs one extra GET that also returns nothing. - if not out: - return None - return out - except Exception as e: - logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}") - return None - - -def _fetch_contacts(force=False): - """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON.""" - if not force and _contact_cache["fetched_at"]: - age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds() - if age < 60: - return _contact_cache["contacts"] - - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - _contact_cache["contacts"] = contacts - _contact_cache["fetched_at"] = datetime.utcnow() - return contacts - - try: - cfg["url"] = _carddav_base_url(cfg) - auth = None - if cfg["username"]: - auth = (cfg["username"], cfg["password"]) - # Preferred path: REPORT gives us hrefs for reliable edit/delete. - contacts = _fetch_via_report(cfg, auth) - if contacts is None: - # Fallback: plain GET, concatenated vCards, no hrefs. - r = httpx.get(cfg["url"], auth=auth, timeout=10) - if r.status_code != 200: - logger.warning(f"CardDAV returned {r.status_code}") - return _contact_cache["contacts"] - contacts = _parse_vcards(r.text) - _contact_cache["contacts"] = contacts - _contact_cache["fetched_at"] = datetime.utcnow() - return contacts - except Exception as e: - logger.error(f"Failed to fetch contacts: {e}") - return _contact_cache["contacts"] - - -def _resolve_resource_url(uid: str) -> str: - """Map a contact UID to its real CardDAV resource URL. Uses the href - captured during fetch when available (handles contacts whose filename - != UID); falls back to the .vcf guess for app-created contacts or - when no href is known.""" - def _lookup(): - for c in _contact_cache.get("contacts", []): - if c.get("uid") == uid and c.get("href"): - return _abs_url(c["href"]) - return None - found = _lookup() - if found: - return found - # Not in cache (or no href) — refresh once and retry before guessing. - try: - _fetch_contacts(force=True) - except Exception: - pass - return _lookup() or _vcard_url(uid) - - -def _create_contact(name: str, email: str, address: str = "") -> bool: - """Add a new contact via CardDAV or local contacts.""" - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - email_l = (email or "").strip().lower() - for c in contacts: - if email_l and email_l in [e.lower() for e in c.get("emails", [])]: - return True - contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address})) - _save_local_contacts(contacts) - return True - - contact_uid = str(uuid.uuid4()) - vcard = _build_vcard(name, email, contact_uid, address=address) - try: - url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf" - auth = None - if cfg["username"]: - auth = (cfg["username"], cfg["password"]) - r = httpx.put( - url, - data=vcard.encode("utf-8"), - headers={"Content-Type": "text/vcard; charset=utf-8"}, - auth=auth, - timeout=10, - ) - if r.status_code in (200, 201, 204): - # Invalidate cache - _contact_cache["fetched_at"] = None - return True - logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}") - return False - except Exception as e: - logger.error(f"Failed to create contact: {e}") - return False - - -def _vcard_url(uid: str) -> str: - """The CardDAV resource URL for a given contact UID. The uid is URL- - encoded so a value containing '/', '..' or other path chars can't - escape the collection and target an arbitrary CardDAV resource.""" - from urllib.parse import quote - cfg = _get_carddav_config() - return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf" - - -def _import_vcards(text: str) -> Dict: - """Import a (possibly multi-card) .vcf blob. Each card is PUT to the - CardDAV server PRESERVING its full original content (ADR/ORG/photo/ - etc.) — we don't rebuild it, just ensure it has VERSION + UID and - normalize line endings. Returns {imported, failed, total}.""" - from urllib.parse import quote - cfg = _get_carddav_config() - if not cfg.get("url"): - parsed = _parse_vcards(text) - contacts = _load_local_contacts() - existing = { - e.lower() - for c in contacts - for e in (c.get("emails") or []) - if e - } - imported = 0 - for c in parsed: - emails = [e for e in (c.get("emails") or []) if e] - if emails and any(e.lower() in existing for e in emails): - continue - contacts.append(_normalize_contact(c)) - for e in emails: - existing.add(e.lower()) - imported += 1 - if imported: - _save_local_contacts(contacts) - return {"imported": imported, "failed": 0, "total": len(parsed)} - try: - base_url = _carddav_base_url(cfg) - except ValueError as e: - logger.warning("CardDAV import URL rejected: %s", e) - return {"imported": 0, "failed": 0, "total": 0, "error": str(e)} - auth = (cfg["username"], cfg["password"]) if cfg["username"] else None - # Split into individual cards. re.split drops the BEGIN line, so we - # re-add it. Normalize CRLF. - raw = (text or "").replace("\r\n", "\n").replace("\r", "\n") - blocks = [] - for chunk in raw.split("BEGIN:VCARD"): - chunk = chunk.strip() - if not chunk: - continue - # Trim anything after END:VCARD (defensive). - end = chunk.upper().find("END:VCARD") - body = chunk[: end + len("END:VCARD")] if end != -1 else chunk - blocks.append("BEGIN:VCARD\n" + body) - imported = 0 - failed = 0 - for block in blocks: - # Extract or assign a UID. - m = re.search(r"^UID:(.+)$", block, re.MULTILINE) - uid = (m.group(1).strip() if m else "") or str(uuid.uuid4()) - if not m: - # Inject a UID right after the VERSION line (or after BEGIN). - if re.search(r"^VERSION:", block, re.MULTILINE): - block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE) - else: - block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1) - elif not re.search(r"^VERSION:", block, re.MULTILINE): - block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1) - vcard = block.replace("\n", "\r\n") + "\r\n" - url = base_url + "/" + quote(uid, safe="") + ".vcf" - try: - r = httpx.put( - url, data=vcard.encode("utf-8"), - headers={"Content-Type": "text/vcard; charset=utf-8"}, - auth=auth, timeout=15, - ) - if r.status_code in (200, 201, 204): - imported += 1 - else: - failed += 1 - logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}") - except Exception as e: - failed += 1 - logger.error(f"Import PUT {uid} failed: {e}") - if imported: - _contact_cache["fetched_at"] = None - return {"imported": imported, "failed": failed, "total": len(blocks)} - - -def _import_csv_contacts(text: str) -> Dict: - """Import contacts from CSV. Supports common headers: - name/full_name/display_name, email/email_address/e-mail, phone/tel. - Falls back to first columns as name,email,phone when no headers exist.""" - raw = (text or "").strip() - if not raw: - return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"} - - try: - sample = raw[:2048] - dialect = csv.Sniffer().sniff(sample) - except Exception: - dialect = csv.excel - - stream = io.StringIO(raw) - try: - has_header = csv.Sniffer().has_header(raw[:2048]) - except Exception: - has_header = True - - rows = [] - if has_header: - reader = csv.DictReader(stream, dialect=dialect) - for row in reader: - lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()} - name = ( - lowered.get("name") or lowered.get("full name") or lowered.get("full_name") - or lowered.get("display name") or lowered.get("display_name") - or lowered.get("fn") or "" - ) - email = ( - lowered.get("email") or lowered.get("email address") - or lowered.get("email_address") or lowered.get("e-mail") - or lowered.get("mail") or "" - ) - phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or "" - rows.append((name, email, phone)) - else: - stream.seek(0) - reader = csv.reader(stream, dialect=dialect) - for row in reader: - cols = [(c or "").strip() for c in row] - if not any(cols): - continue - rows.append(( - cols[0] if len(cols) > 0 else "", - cols[1] if len(cols) > 1 else "", - cols[2] if len(cols) > 2 else "", - )) - - imported = 0 - failed = 0 - total = 0 - existing_emails = { - e.lower() - for c in _fetch_contacts() - for e in (c.get("emails") or []) - if e - } - for name, email, phone in rows: - email = (email or "").strip() - name = (name or "").strip() or (email.split("@")[0] if email else "") - if not email: - continue - total += 1 - if email.lower() in existing_emails: - continue - ok = _create_contact(name, email) - if ok: - imported += 1 - existing_emails.add(email.lower()) - # If the CSV had a phone number, rewrite the just-created row - # through the richer update path so phone lands in CardDAV too. - if phone: - try: - contacts = _fetch_contacts(force=True) - created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None) - if created and created.get("uid"): - _update_contact(created["uid"], name, [email], [phone]) - except Exception: - pass - else: - failed += 1 - - if imported: - _contact_cache["fetched_at"] = None - return {"imported": imported, "failed": failed, "total": total} - - -def _contacts_to_vcf(contacts: List[Dict]) -> str: - return "".join( - _build_vcard( - c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"), - "", - uid=c.get("uid") or str(uuid.uuid4()), - emails=c.get("emails") or [], - phones=c.get("phones") or [], - ) - for c in contacts - ) - - -def _contacts_to_csv(contacts: List[Dict]) -> str: - out = io.StringIO() - writer = csv.writer(out) - writer.writerow(["name", "email", "phone"]) - for c in contacts: - emails = c.get("emails") or [""] - phones = c.get("phones") or [""] - max_len = max(len(emails), len(phones), 1) - for i in range(max_len): - writer.writerow([ - c.get("name") or "", - emails[i] if i < len(emails) else "", - phones[i] if i < len(phones) else "", - ]) - return out.getvalue() - - -def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool: - """Rewrite an existing contact via CardDAV or local contacts.""" - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - found = False - out = [] - for c in contacts: - if c.get("uid") == uid: - # Preserve existing address when caller passes "" (only - # updating name/emails/phones, not touching address). - addr = address if address else c.get("address", "") - out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr})) - found = True - else: - out.append(c) - if not found: - out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address})) - _save_local_contacts(out) - return True - - vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address) - # Use the real resource href (handles externally-created contacts whose - # filename != UID); falls back to the .vcf guess. - try: - url = _resolve_resource_url(uid) - auth = (cfg["username"], cfg["password"]) if cfg["username"] else None - r = httpx.put( - url, - data=vcard.encode("utf-8"), - headers={"Content-Type": "text/vcard; charset=utf-8"}, - auth=auth, - timeout=10, - ) - if r.status_code in (200, 201, 204): - _contact_cache["fetched_at"] = None - return True - logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}") - return False - except Exception as e: - logger.error(f"Failed to update contact: {e}") - return False - - -def _delete_contact(uid: str) -> bool: - """Delete a contact via CardDAV or local contacts.""" - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - remaining = [c for c in contacts if c.get("uid") != uid] - _save_local_contacts(remaining) - return True - - try: - url = _resolve_resource_url(uid) - auth = (cfg["username"], cfg["password"]) if cfg["username"] else None - r = httpx.delete(url, auth=auth, timeout=10) - if r.status_code in (200, 204, 404): - # Invalidate cache so the next fetch sees the server truth. - _contact_cache["fetched_at"] = None - # Verify: force a fresh fetch and check the UID is actually gone. - # A 404 on the guessed URL ({uid}.vcf) can mean the contact - # lives at a different resource URL — the DELETE missed it but - # we'd silently report success. This check catches that. - fresh = _fetch_contacts(force=True) - still_there = any(c.get("uid") == uid for c in fresh) - if still_there: - logger.warning( - f"CardDAV DELETE reported success for {uid} " - f"but UID still present after re-fetch — " - f"resource URL may differ from {redact_url(url)}" - ) - return False - if r.status_code == 404: - logger.info(f"CardDAV DELETE 404 for {uid} — already gone") - return True - logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}") - return False - except Exception as e: - logger.error(f"Failed to delete contact: {e}") - return False - - -# ── Routes ── - -def setup_contacts_routes(): - router = APIRouter(prefix="/api/contacts", tags=["contacts"]) - - @router.get("/list") - async def list_contacts(_admin: str = Depends(require_admin)): - """List all contacts.""" - contacts = _fetch_contacts() - return {"contacts": contacts, "count": len(contacts)} - - @router.get("/search") - async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)): - """Search contacts by name or email. Returns up to 10 matches.""" - contacts = _fetch_contacts() - if not q: - return {"results": []} - q_lower = q.lower() - results = [] - for c in contacts: - if q_lower in c["name"].lower(): - results.append(c) - continue - for em in c["emails"]: - if q_lower in em.lower(): - results.append(c) - break - return {"results": results[:10]} - - @router.post("/add") - async def add_contact(data: dict, _admin: str = Depends(require_admin)): - """Add a new contact.""" - name = (data.get("name") or "").strip() - email = (data.get("email") or "").strip() - phone = (data.get("phone") or "").strip() - address = (data.get("address") or "").strip() - if not email: - return {"success": False, "error": "Email required"} - # Check if already exists by email - if email: - contacts = _fetch_contacts() - for c in contacts: - if email.lower() in [e.lower() for e in c["emails"]]: - return {"success": True, "message": "Already exists", "contact": c} - if not name: - name = email.split("@")[0] - create_params = inspect.signature(_create_contact).parameters - if len(create_params) >= 3: - ok = _create_contact(name, email, address) - else: - ok = _create_contact(name, email) - # If a phone was provided, do an immediate update to thread it - # through (the simple _create_contact signature only takes name + - # email + address; phones happen via update). - if ok and phone: - try: - fresh = _fetch_contacts(force=True) - created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None) - if created: - _update_contact( - created["uid"], name, - created.get("emails", []), - [phone], - address, - ) - except Exception: - pass - return {"success": ok} - - @router.post("/import") - async def import_vcf(data: dict, _admin: str = Depends(require_admin)): - """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}.""" - # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list - # in the JSON body) would otherwise reach .strip() and 500 with an - # AttributeError instead of degrading to a clean "no data" response. - text = str(data.get("vcf") or data.get("text") or "") - csv_text = str(data.get("csv") or "") - if text.strip(): - if "BEGIN:VCARD" not in text.upper(): - return {"success": False, "error": "No vCard data found"} - result = _import_vcards(text) - elif csv_text.strip(): - result = _import_csv_contacts(csv_text) - else: - return {"success": False, "error": "No contact data found"} - result["success"] = result.get("imported", 0) > 0 - return result - - @router.get("/export") - async def export_contacts( - format: str = Query("vcf", pattern="^(vcf|csv)$"), - _admin: str = Depends(require_admin), - ): - """Export all contacts as vCard or CSV.""" - contacts = _fetch_contacts(force=True) - if format == "csv": - content = _contacts_to_csv(contacts) - media_type = "text/csv; charset=utf-8" - filename = "odysseus-contacts.csv" - else: - content = _contacts_to_vcf(contacts) - media_type = "text/vcard; charset=utf-8" - filename = "odysseus-contacts.vcf" - return Response( - content=content, - media_type=media_type, - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - @router.get("/config") - async def get_config(_admin: str = Depends(require_admin)): - cfg = _get_carddav_config() - # Mask password - if cfg["password"]: - cfg["password"] = "***" - return cfg - - @router.put("/config") - async def update_config(data: dict, _admin: str = Depends(require_admin)): - settings = _load_settings() - for key in ("carddav_url", "carddav_username", "carddav_password"): - if key in data: - if key == "carddav_url" and str(data[key] or "").strip(): - try: - settings[key] = _validate_carddav_url(data[key]) - except ValueError as e: - raise HTTPException(400, str(e)) - else: - value = data[key] - if key == "carddav_password" and value: - from src.secret_storage import encrypt - value = encrypt(value) - settings[key] = value - _save_settings(settings) - # Force re-fetch - _contact_cache["fetched_at"] = None - return {"success": True} - - @router.delete("/clear") - async def clear_contacts(_admin: str = Depends(require_admin)): - """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache.""" - _save_local_contacts([]) - return {"success": True} - - # NOTE: the /{uid} routes are declared LAST so the literal paths above - # (/list, /search, /add, /config) win — otherwise PUT /config would - # match PUT /{uid} with uid="config". - @router.put("/{uid}") - async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)): - """Edit an existing contact — name / emails / phones / address.""" - name = (data.get("name") or "").strip() - emails = data.get("emails") - phones = data.get("phones") - if emails is None and data.get("email"): - emails = [data["email"]] - emails = [e.strip() for e in (emails or []) if e and e.strip()] - phones = [p.strip() for p in (phones or []) if p and p.strip()] - address = (data.get("address") or "").strip() - if not name and not emails and not address: - return {"success": False, "error": "Name, email, or address required"} - if not name and emails: - name = emails[0].split("@")[0] - ok = _update_contact(uid, name, emails, phones, address) - return {"success": ok} - - @router.delete("/{uid}") - async def delete_contact(uid: str, _admin: str = Depends(require_admin)): - """Delete a contact by UID.""" - if not uid: - return {"success": False, "error": "UID required"} - ok = _delete_contact(uid) - return {"success": ok} - - return router +_sys.modules[__name__] = _canonical diff --git a/tests/test_contacts_routes_shim.py b/tests/test_contacts_routes_shim.py new file mode 100644 index 000000000..1e52f9f34 --- /dev/null +++ b/tests/test_contacts_routes_shim.py @@ -0,0 +1,46 @@ +"""Regression test for the contacts route shim (slice 2e, #4082/#4071). + +The backward-compat shim at ``routes/contacts_routes.py`` uses ``sys.modules`` +replacement so the legacy import path and the canonical ``routes.contacts.*`` +path resolve to the *same* module object. This is required because: + +* ``test_carddav_password_encryption.py`` uses string-targeted + ``monkeypatch.setattr("routes.contacts_routes.SETTINGS_FILE", ...)`` which + must reach the canonical module to take effect; +* ``test_contacts_add_null_name.py`` / ``test_contacts_carddav_security.py`` + use ``import routes.contacts_routes as cr`` + ``setattr(cr, ...)``; +* the module owns mutable state (``_contact_cache``) that must be shared + across import paths. +""" + +import importlib + +import routes.contacts_routes as _shim_contacts # noqa: F401 + + +def test_legacy_and_canonical_contacts_module_are_same_object(): + """``import routes.contacts_routes`` must alias the canonical module.""" + legacy = importlib.import_module("routes.contacts_routes") + canonical = importlib.import_module("routes.contacts.contacts_routes") + assert legacy is canonical, ( + "routes.contacts_routes shim must resolve to the canonical " + "routes.contacts.contacts_routes module object" + ) + + +def test_string_targeted_monkeypatch_reaches_canonical(monkeypatch): + """String-targeted ``monkeypatch.setattr`` via the legacy path must reach + the canonical module. + + ``test_carddav_password_encryption.py`` patches + ``"routes.contacts_routes.SETTINGS_FILE"`` as a fixture setup; for that + to take effect at runtime, the legacy module name and the canonical + module must be identical. + """ + canonical = importlib.import_module("routes.contacts.contacts_routes") + + sentinel = object() + monkeypatch.setattr("routes.contacts_routes.setup_contacts_routes", sentinel) + assert canonical.setup_contacts_routes is sentinel, ( + "string-targeted monkeypatch via legacy path did not reach the canonical module" + ) From c8169ad7a9211ddde38b5c44ca790c6a458fd0e8 Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Sun, 5 Jul 2026 17:20:32 +0530 Subject: [PATCH 30/32] fix(security): scope owner-less email accounts to a mailbox match in route guards (#5238) The HTTP email route guard `_assert_owns_account` and the explicit-account_id path in `_get_email_config` gated cross-tenant access with `if row.owner and row.owner != owner` -- which skips the check entirely when the account row is owner-less (owner NULL or ""). `email_accounts` is the one owner-scoped table left out of the legacy-owner migration backfill (core/database.py), so such rows persist on multi-user deploys: an account configured while auth was disabled, or an imported legacy row. Any authenticated user could then pass that account's id to read/send/update-credentials/delete another tenant's mailbox and read its decrypted IMAP/SMTP creds. Both sibling paths already enforce the intended contract -- the same-file `_owner_or_matching_legacy_account` fallback and the MCP `_account_visible_to_owner` gate (whose comment says it mirrors "the HTTP email route fallback") only expose an owner-less account when its own mailbox (imap_user / from_address) is the caller's. Factor that row-level predicate into `_account_visible_to_owner` and use it in both guards, so owner-less accounts are visible only on a mailbox match. Owned accounts, the legacy-claim path, and single-user mode (owner == "") are unchanged. Complements #5234 (which fixes the same class on the MCP tool layer); this is the HTTP route layer it does not touch. --- routes/email_helpers.py | 29 ++++- ...est_email_ownerless_account_owner_scope.py | 117 ++++++++++++++++++ 2 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 tests/test_email_ownerless_account_owner_scope.py diff --git a/routes/email_helpers.py b/routes/email_helpers.py index b05f64681..b04be6066 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -349,7 +349,7 @@ def _assert_owns_account(account_id: str, owner: str) -> None: row = db.query(_EA).filter(_EA.id == account_id).first() if row is None: raise HTTPException(404, "Account not found") - if row.owner and row.owner != owner: + if not _account_visible_to_owner(row, owner): # Treat as 404 (not 403) so we don't leak existence. raise HTTPException(404, "Account not found") finally: @@ -362,6 +362,26 @@ def _assert_owns_account(account_id: str, owner: str) -> None: logger.error(f"Account-owner check failed: {e}") raise HTTPException(503, "Account check failed") + +def _account_visible_to_owner(row, owner: str) -> bool: + """Whether an authenticated `owner` may act on this EmailAccount row. + + Mirrors the SQL predicate in `_get_email_config`'s + `_owner_or_matching_legacy_account`: a caller sees an account they own, or a + legacy owner-less account (owner NULL/"") only when its own mailbox + (`imap_user` / `from_address`) is the caller's. `email_accounts` is the one + owner-scoped table deliberately left out of the legacy-owner migration + backfill, so ownerless rows persist on multi-user deploys — making this the + gate that keeps one tenant off another's imported mailbox and its decrypted + IMAP/SMTP credentials.""" + row_owner = getattr(row, "owner", None) or "" + if row_owner: + return row_owner == owner + return owner in { + getattr(row, "imap_user", None) or "", + getattr(row, "from_address", None) or "", + } + def _q(name: str) -> str: """Quote an IMAP mailbox name. Defensive: escapes `\\` and `"` and wraps in double quotes so user-supplied folder names with spaces or quotes can't @@ -903,12 +923,13 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict: try: if account_id: row = db.query(_EA).filter(_EA.id == account_id, _EA.enabled == True).first() # noqa: E712 - # If the resolved row belongs to a different owner, treat as + # If the resolved row isn't visible to this owner, treat as # not-found rather than silently serving it. This is a defense # in depth — `require_owner` already calls `_assert_owns_account` # for query-param account_ids, but other callers (cookbook - # rules, scheduled poller) may not. - if row is not None and owner and row.owner and row.owner != owner: + # rules, scheduled poller) may not. Ownerless legacy rows are + # only visible on a mailbox match, same as the fallback below. + if row is not None and owner and not _account_visible_to_owner(row, owner): row = None # Fallback path — restrict to this owner's accounts so we don't # leak another user's default mailbox to an unconfigured user. diff --git a/tests/test_email_ownerless_account_owner_scope.py b/tests/test_email_ownerless_account_owner_scope.py new file mode 100644 index 000000000..bf18dc02a --- /dev/null +++ b/tests/test_email_ownerless_account_owner_scope.py @@ -0,0 +1,117 @@ +"""Cross-tenant access control for legacy owner-less email accounts. + +`email_accounts` is the one owner-scoped table left out of the legacy-owner +migration backfill (core/database.py), so rows with owner NULL/"" persist on a +multi-user deploy — e.g. an account configured while auth was disabled, or an +imported legacy row. The HTTP route guards (`_assert_owns_account` and the +explicit-account_id path in `_get_email_config`) must scope such rows to a +mailbox match, exactly like the `_owner_or_matching_legacy_account` fallback and +the MCP `_account_visible_to_owner` gate. Otherwise any authenticated user can +read/send/update-credentials/delete another tenant's imported mailbox. +""" + +from unittest import mock + +import pytest +from fastapi import HTTPException + + +def _make_db(): + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + from core.database import Base + engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}) + Base.metadata.create_all(engine) + Factory = sessionmaker(bind=engine) + return Factory + + +def _make_account(Factory, account_id, owner, imap_user, from_address="", is_default=False): + from core.database import EmailAccount + db = Factory() + row = EmailAccount( + id=account_id, + owner=owner, + name="Test", + enabled=True, + is_default=is_default, + imap_host="imap.example.com", + imap_port=993, + imap_user=imap_user, + smtp_host="smtp.example.com", + smtp_port=587, + smtp_user=imap_user, + from_address=from_address or imap_user, + ) + db.add(row) + db.commit() + db.close() + + +def test_assert_owns_account_rejects_ownerless_account_for_other_tenant(): + """The core regression: a legacy owner-less mailbox is NOT accessible to an + authenticated caller whose own mailbox does not match it.""" + from routes.email_helpers import _assert_owns_account + Factory = _make_db() + # owner="" (created while auth was disabled); mailbox belongs to victim. + _make_account(Factory, "acct-legacy", owner="", imap_user="victim@corp.com") + + with mock.patch("core.database.SessionLocal", Factory): + with pytest.raises(HTTPException) as exc: + _assert_owns_account("acct-legacy", "attacker") + assert exc.value.status_code == 404 + + +def test_assert_owns_account_allows_owned_account(): + from routes.email_helpers import _assert_owns_account + Factory = _make_db() + _make_account(Factory, "acct-bob", owner="bob", imap_user="bob@corp.com") + with mock.patch("core.database.SessionLocal", Factory): + _assert_owns_account("acct-bob", "bob") # no raise + + +def test_assert_owns_account_allows_ownerless_account_on_mailbox_match(): + """Legacy-claim path stays intact: the user whose mailbox matches an + owner-less account may still act on it (imap_user or from_address).""" + from routes.email_helpers import _assert_owns_account + Factory = _make_db() + _make_account(Factory, "acct-legacy", owner="", imap_user="alice@corp.com") + with mock.patch("core.database.SessionLocal", Factory): + _assert_owns_account("acct-legacy", "alice@corp.com") # no raise + + +def test_assert_owns_account_noop_for_single_user_mode(): + """owner == "" (unconfigured / single-user) accepts any account, unchanged.""" + from routes.email_helpers import _assert_owns_account + Factory = _make_db() + _make_account(Factory, "acct-legacy", owner="", imap_user="whoever@corp.com") + with mock.patch("core.database.SessionLocal", Factory): + _assert_owns_account("acct-legacy", "") # no raise + + +def test_get_email_config_does_not_resolve_ownerless_account_for_other_tenant(monkeypatch): + """`_get_email_config(account_id=..., owner=...)` must not serve an + owner-less account (and its decrypted creds) to a non-matching tenant.""" + import routes.email_helpers as eh + Factory = _make_db() + _make_account(Factory, "acct-legacy", owner="", imap_user="victim@corp.com", is_default=True) + + # Make the settings.json / env fallback empty and deterministic. + monkeypatch.setattr(eh, "_load_settings", lambda: {}, raising=False) + for var in ("IMAP_HOST", "SMTP_HOST", "IMAP_USER", "SMTP_USER"): + monkeypatch.delenv(var, raising=False) + + with mock.patch("core.database.SessionLocal", Factory): + cfg = eh._get_email_config(account_id="acct-legacy", owner="attacker") + + assert cfg.get("account_id") != "acct-legacy" + + +def test_get_email_config_resolves_ownerless_account_on_mailbox_match(): + """The mailbox owner still resolves their claimable legacy account by id.""" + import routes.email_helpers as eh + Factory = _make_db() + _make_account(Factory, "acct-legacy", owner="", imap_user="alice@corp.com") + with mock.patch("core.database.SessionLocal", Factory): + cfg = eh._get_email_config(account_id="acct-legacy", owner="alice@corp.com") + assert cfg.get("account_id") == "acct-legacy" From 3592285db74af10008fa13e7a69cd2f7ecfc2bdc Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:13:56 +0200 Subject: [PATCH 31/32] fix(email): enforce MCP account owner scope (#5234) --- mcp_servers/email_server.py | 44 +++++-- tests/test_mcp_email_decode_header_spaces.py | 121 ++++++++++++++++++- 2 files changed, 149 insertions(+), 16 deletions(-) diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index 96c985864..e2bccfbfb 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -58,6 +58,11 @@ def _uid_fetch_rows(data) -> list: _ACCOUNT_CACHE: dict = {} # key = normalized account selector -> config dict _MCP_OWNER_ARG = "_odysseus_owner" _CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None) +_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_EMAIL_OWNER", "ODYSSEUS_EMAIL_OWNER") +_OWNER_SCOPE_ERROR = ( + "Error: email MCP requires an authenticated owner or ODYSSEUS_MCP_EMAIL_OWNER " + "when owner-scoped email accounts are configured." +) def _clean_header_value(value) -> str: @@ -71,13 +76,29 @@ def _db_path() -> Path: return Path(APP_DB) +def _configured_owner() -> str | None: + for key in _OWNER_ENV_KEYS: + owner = os.environ.get(key, "").strip() + if owner: + return owner + return None + + def _current_owner() -> str: owner = _CURRENT_OWNER.get() - return str(owner or "").strip() + return str(owner or _configured_owner() or "").strip() + + +def _account_owner(row: dict) -> str: + return str(row.get("owner") or "").strip() + + +def _has_owner_scoped_accounts(rows: list[dict]) -> bool: + return any(_account_owner(r) for r in rows) def _account_visible_to_owner(row: dict, owner: str) -> bool: - row_owner = str(row.get("owner") or "").strip() + row_owner = _account_owner(row) if row_owner == owner: return True if row_owner: @@ -96,8 +117,7 @@ def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]: if owner: return [r for r in rows if _account_visible_to_owner(r, owner)] - owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()} - if len(owners) > 1: + if _has_owner_scoped_accounts(rows): return [] return rows @@ -106,8 +126,7 @@ def _mcp_owner_required(rows: list[dict] | None = None) -> bool: if _current_owner(): return False rows = rows if rows is not None else _read_accounts_from_db() - owners = {str(r.get("owner") or "").strip() for r in rows if str(r.get("owner") or "").strip()} - return len(owners) > 1 + return _has_owner_scoped_accounts(rows) def _load_email_writing_style() -> str: @@ -274,6 +293,8 @@ def _load_config(account: str | None = None) -> dict: } raw_rows = _read_accounts_from_db() + if _mcp_owner_required(raw_rows): + raise ValueError(_OWNER_SCOPE_ERROR) rows = _filter_accounts_for_owner(raw_rows) row = _resolve_account_from_rows(rows, account) if _current_owner() and raw_rows and not rows: @@ -1193,10 +1214,14 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b UI. This closes the auto-send hole that let earlier models invent signatures and ship them to real recipients without confirmation.""" if _read_agent_email_confirm_setting(): + # Even confirmation-first sends must resolve the selected account now. + # Otherwise a caller could stage a pending draft against another + # owner's account selector before browser approval handles it. + cfg = _load_config(account) return _stash_agent_draft( to=to, subject=subject, body=body, in_reply_to=in_reply_to, references=references, - cc=cc, bcc=bcc, account=account, + cc=cc, bcc=bcc, account=cfg.get("account_id") or account, ) send_account, cfg = _resolve_send_config(account) msg = EmailMessage() @@ -2142,10 +2167,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: try: all_db_accounts = _read_accounts_from_db() if _mcp_owner_required(all_db_accounts): - return [TextContent( - type="text", - text="Error: email MCP requires an authenticated owner when multiple email account owners are configured.", - )] + return [TextContent(type="text", text=_OWNER_SCOPE_ERROR)] if name == "list_email_accounts": rows = _filter_accounts_for_owner(all_db_accounts) diff --git a/tests/test_mcp_email_decode_header_spaces.py b/tests/test_mcp_email_decode_header_spaces.py index f588a6bd3..6370801b0 100644 --- a/tests/test_mcp_email_decode_header_spaces.py +++ b/tests/test_mcp_email_decode_header_spaces.py @@ -16,7 +16,20 @@ pytest.importorskip("mcp") import mcp_servers.email_server as es -def _init_accounts_db(path): +@pytest.fixture(autouse=True) +def _clear_mcp_email_owner_env(monkeypatch): + for key in es._OWNER_ENV_KEYS: + monkeypatch.delenv(key, raising=False) + es._ACCOUNT_CACHE.clear() + yield + es._ACCOUNT_CACHE.clear() + + +def _init_accounts_db(path, rows=None): + rows = rows or [ + ("acct-alice", "alice", "Alice Mail", 1, "alice@example.com", "alice@example.com", "alice@example.com", "2026-01-01"), + ("acct-bob", "bob", "Bob Mail", 1, "bob@example.com", "bob@example.com", "bob@example.com", "2026-01-02"), + ] conn = sqlite3.connect(path) conn.execute( """ @@ -50,10 +63,7 @@ def _init_accounts_db(path): VALUES (?, ?, ?, ?, 1, 'imap.example.com', 993, ?, '', 1, 'smtp.example.com', 465, 'ssl', ?, '', ?, ?) """, - [ - ("acct-alice", "alice", "Alice Mail", 1, "alice@example.com", "alice@example.com", "alice@example.com", "2026-01-01"), - ("acct-bob", "bob", "Bob Mail", 1, "bob@example.com", "bob@example.com", "bob@example.com", "2026-01-02"), - ], + rows, ) conn.commit() conn.close() @@ -106,6 +116,37 @@ async def test_mcp_email_requires_owner_when_multiple_account_owners_exist(tmp_p assert "requires an authenticated owner" in out[0].text +@pytest.mark.asyncio +async def test_mcp_email_requires_owner_when_any_account_owner_exists(tmp_path, monkeypatch): + db_path = tmp_path / "app.db" + _init_accounts_db( + db_path, + rows=[ + ("acct-alice", "alice", "Alice Mail", 1, "alice@example.com", "alice@example.com", "alice@example.com", "2026-01-01"), + ], + ) + monkeypatch.setattr(es, "APP_DB", str(db_path)) + + out = await es.call_tool("list_email_accounts", {}) + + assert "requires an authenticated owner" in out[0].text + assert "Alice Mail" not in out[0].text + + +@pytest.mark.asyncio +async def test_mcp_email_configured_owner_filters_accounts(tmp_path, monkeypatch): + db_path = tmp_path / "app.db" + _init_accounts_db(db_path) + monkeypatch.setattr(es, "APP_DB", str(db_path)) + monkeypatch.setenv("ODYSSEUS_MCP_EMAIL_OWNER", "alice") + + out = await es.call_tool("list_email_accounts", {}) + text = out[0].text + + assert "Alice Mail" in text + assert "Bob Mail" not in text + + def test_mcp_email_scoped_owner_without_visible_account_skips_legacy_fallback(tmp_path, monkeypatch): db_path = tmp_path / "app.db" settings_path = tmp_path / "settings.json" @@ -137,11 +178,81 @@ def test_mcp_email_scoped_owner_without_visible_account_skips_legacy_fallback(tm es._ACCOUNT_CACHE.clear() +@pytest.mark.asyncio +async def test_mcp_email_owner_cannot_use_other_owner_account_for_list_read_send_or_draft(tmp_path, monkeypatch): + import src.constants as constants + + db_path = tmp_path / "app.db" + scheduled_path = tmp_path / "scheduled_emails.db" + _init_accounts_db(db_path) + monkeypatch.setattr(es, "APP_DB", str(db_path)) + monkeypatch.setattr(constants, "SCHEDULED_EMAILS_DB", str(scheduled_path)) + monkeypatch.setattr(es, "_read_agent_email_confirm_setting", lambda: True) + + calls = [ + ("list_emails", {"account": "Bob Mail"}), + ("read_email", {"uid": "1", "account": "Bob Mail"}), + ("send_email", { + "to": "recipient@example.com", + "subject": "Blocked", + "body": "Do not stage.", + "account": "Bob Mail", + }), + ("draft_email", { + "to": "recipient@example.com", + "subject": "Blocked", + "body": "Do not draft.", + "account": "Bob Mail", + }), + ] + + for tool_name, args in calls: + out = await es.call_tool(tool_name, {**args, "_odysseus_owner": "alice"}) + assert "Email account not found for selector" in out[0].text, tool_name + assert "Bob Mail" not in out[0].text or "Available accounts" in out[0].text + + assert not scheduled_path.exists() + + +@pytest.mark.asyncio +async def test_mcp_send_email_stages_with_visible_owner_account_id(tmp_path, monkeypatch): + import src.constants as constants + + app_db_path = tmp_path / "app.db" + scheduled_path = tmp_path / "scheduled_emails.db" + _init_accounts_db(app_db_path) + monkeypatch.setattr(es, "APP_DB", str(app_db_path)) + monkeypatch.setattr(constants, "SCHEDULED_EMAILS_DB", str(scheduled_path)) + monkeypatch.setattr(es, "_read_agent_email_confirm_setting", lambda: True) + + out = await es.call_tool( + "send_email", + { + "to": "recipient@example.com", + "subject": "Review", + "body": "Please review.", + "account": "Alice Mail", + "_odysseus_owner": "alice", + }, + ) + + assert "Draft staged for approval" in out[0].text + conn = sqlite3.connect(scheduled_path) + try: + row = conn.execute( + "SELECT owner, status, account_id FROM scheduled_emails" + ).fetchone() + finally: + conn.close() + assert row == ("alice", "agent_draft", "acct-alice") + + @pytest.mark.asyncio async def test_mcp_send_email_stages_owner_scoped_pending_draft(tmp_path, monkeypatch): import src.constants as constants db_path = tmp_path / "scheduled_emails.db" + monkeypatch.setattr(es, "APP_DB", str(tmp_path / "missing-app.db")) monkeypatch.setattr(constants, "SCHEDULED_EMAILS_DB", str(db_path)) monkeypatch.setattr(es, "_read_agent_email_confirm_setting", lambda: True) From 2826dcfc3300cb75f93af91d214ab80e0607d399 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:19:04 +0200 Subject: [PATCH 32/32] fix(tasks): gate cookbook serve task execution (#5235) --- routes/task_routes.py | 52 ++-- src/task_action_policy.py | 47 ++++ src/task_scheduler.py | 26 ++ tests/test_auth_regressions.py | 15 +- tests/test_task_cookbook_admin_gate.py | 350 +++++++++++++++++++++++++ 5 files changed, 456 insertions(+), 34 deletions(-) create mode 100644 src/task_action_policy.py create mode 100644 tests/test_task_cookbook_admin_gate.py diff --git a/routes/task_routes.py b/routes/task_routes.py index fb1a7c746..d786c5730 100644 --- a/routes/task_routes.py +++ b/routes/task_routes.py @@ -11,10 +11,14 @@ from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel from core.database import SessionLocal, ScheduledTask, TaskRun -from core.middleware import INTERNAL_TOOL_USER from core.constants import internal_api_base from src.auth_helpers import get_current_user from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR +from src.task_action_policy import ( + ADMIN_ONLY_TASK_ACTIONS, + is_admin_only_task_action, + owner_has_admin_task_privileges, +) from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS from routes.prefs_routes import _load_for_user, _save_for_user @@ -417,28 +421,18 @@ def setup_task_routes(task_scheduler) -> APIRouter: db.close() return {"ok": True, "opened": True, "enabled": bool(prefs.get("tasks_enabled")), "resumed": resumed} - # Actions that execute shell/SSH commands — restricted to admins. + # Actions that execute shell/SSH commands or cross into admin-only + # Cookbook serving surfaces — restricted to admins. # Non-admin users cannot create tasks with these action types via the # API. See review CRIT-C. - _ADMIN_ONLY_ACTIONS = {"run_local", "run_script", "ssh_command"} + _ADMIN_ONLY_ACTIONS = ADMIN_ONLY_TASK_ACTIONS def _is_admin(user: str | None) -> bool: - if not user: - return False - # In-process tool-loopback marker — AuthMiddleware validated - # the internal token + loopback client before stamping this, - # so treat as admin-equivalent. - if user == INTERNAL_TOOL_USER: - return True - try: - from core.auth import AuthManager - auth = AuthManager() - if not auth.is_configured: - # Unconfigured single-user deploy: trust the local owner. - return True - return bool(auth.is_admin(user)) - except Exception: - return False + return owner_has_admin_task_privileges(user) + + def _require_admin_for_task_action(user: str | None, task_type: str | None, action: str | None) -> None: + if is_admin_only_task_action(task_type, action) and not _is_admin(user): + raise HTTPException(403, f"Action '{action}' requires admin privileges") def _validate_then_task_id(db, then_task_id: Optional[str], user: Optional[str], current_task_id: Optional[str] = None) -> Optional[str]: target_id = (then_task_id or "").strip() @@ -466,8 +460,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: # Block shell-executing action types for non-admins. action_run_local # uses subprocess.run(shell=True) and ssh_command / run_script run # arbitrary commands. - if req.task_type == "action" and req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user): - raise HTTPException(403, f"Action '{req.action}' requires admin privileges") + _require_admin_for_task_action(user, req.task_type, req.action) if req.trigger_type == "schedule" and not req.schedule: raise HTTPException(400, "Schedule is required for schedule-triggered tasks") if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression: @@ -681,6 +674,10 @@ def setup_task_routes(task_scheduler) -> APIRouter: if user and task.owner != user: raise HTTPException(403, "Access denied") + next_task_type = req.task_type if req.task_type is not None else task.task_type + next_action = req.action if req.action is not None else task.action + _require_admin_for_task_action(user, next_task_type, next_action) + if req.name is not None: task.name = req.name if req.prompt is not None: @@ -688,9 +685,6 @@ def setup_task_routes(task_scheduler) -> APIRouter: if req.task_type is not None: task.task_type = req.task_type if req.action is not None: - # Same admin-only gate as create — see CRIT-C. - if req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user): - raise HTTPException(403, f"Action '{req.action}' requires admin privileges") task.action = req.action if req.output_target is not None: task.output_target = req.output_target @@ -807,6 +801,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: raise HTTPException(404, "Task not found") if user and task.owner != user: raise HTTPException(403, "Access denied") + _require_admin_for_task_action(user, task.task_type, task.action) task.status = "active" if (task.trigger_type or "schedule") == "schedule": task.next_run = compute_next_run( @@ -869,6 +864,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: raise HTTPException(404, "Task not found") if user and task.owner != user: raise HTTPException(403, "Access denied") + _require_admin_for_task_action(user, task.task_type, task.action) finally: db.close() started = await task_scheduler.run_task_now(task_id, force=force) @@ -1058,6 +1054,14 @@ def setup_task_routes(task_scheduler) -> APIRouter: ).first() if not task: raise HTTPException(404, "Not found") + if ( + is_admin_only_task_action(task.task_type, task.action) + and not owner_has_admin_task_privileges(task.owner) + ): + task.status = "paused" + task.next_run = None + db.commit() + raise HTTPException(403, f"Action '{task.action}' requires admin privileges") finally: db.close() started = await task_scheduler.run_task_now(task_id) diff --git a/src/task_action_policy.py b/src/task_action_policy.py new file mode 100644 index 000000000..76ae1337e --- /dev/null +++ b/src/task_action_policy.py @@ -0,0 +1,47 @@ +"""Shared privilege policy for scheduled task actions.""" + +from __future__ import annotations + +ADMIN_ONLY_TASK_ACTIONS = frozenset({ + "run_local", + "run_script", + "ssh_command", + "cookbook_serve", +}) + + +def is_admin_only_task_action(task_type: str | None, action: str | None) -> bool: + return (task_type or "llm") == "action" and (action or "") in ADMIN_ONLY_TASK_ACTIONS + + +def owner_has_admin_task_privileges(owner: str | None) -> bool: + try: + from src.auth_helpers import _auth_disabled + if _auth_disabled(): + return True + except Exception: + pass + + if owner: + try: + from core.middleware import INTERNAL_TOOL_USER + if owner == INTERNAL_TOOL_USER: + return True + except Exception: + pass + + try: + from core.auth import AuthManager + auth = AuthManager() + if not auth.is_configured: + return True + if not owner: + return False + return bool(auth.is_admin(owner)) + except Exception: + pass + + if not owner: + return False + + return False diff --git a/src/task_scheduler.py b/src/task_scheduler.py index df515ebea..878edfbea 100644 --- a/src/task_scheduler.py +++ b/src/task_scheduler.py @@ -10,6 +10,10 @@ from datetime import datetime, timedelta, timezone from typing import Any, Awaitable, Callable, Dict, Tuple from core.auth import RESERVED_USERNAMES +from src.task_action_policy import ( + is_admin_only_task_action, + owner_has_admin_task_privileges, +) logger = logging.getLogger(__name__) @@ -821,6 +825,28 @@ class TaskScheduler: db.commit() return + if ( + is_admin_only_task_action(task.task_type, task.action) + and not owner_has_admin_task_privileges(task.owner) + ): + msg = f"Action '{task.action}' requires admin privileges" + blocked = db.query(TaskRun).filter(TaskRun.id == run_id).first() + if blocked: + blocked.status = "error" + blocked.result = msg + blocked.error = msg + blocked.finished_at = _utcnow() + task.status = "paused" + task.next_run = None + task.last_run = _utcnow() + logger.warning( + "Paused admin-only task %s for non-admin owner %r", + task_id, + task.owner, + ) + db.commit() + return + if gate_foreground: waiting = db.query(TaskRun).filter(TaskRun.id == run_id).first() if waiting and waiting.status == "queued": diff --git a/tests/test_auth_regressions.py b/tests/test_auth_regressions.py index b16966e3a..62b479748 100644 --- a/tests/test_auth_regressions.py +++ b/tests/test_auth_regressions.py @@ -334,16 +334,11 @@ def test_pop_notifications_owner_filtered(): def test_admin_only_actions_set_contains_shell_runners(): """The constant defining shell-executing action types must include the three risky entries. Catches accidental removal.""" - from routes import task_routes - # `_ADMIN_ONLY_ACTIONS` is a closure constant. Easiest pin: re-read - # the source and check for the three risky entries + the admin gate - # wording. - src = open(task_routes.__file__, encoding="utf-8").read() - assert '"run_local"' in src - assert '"run_script"' in src - assert '"ssh_command"' in src - # And the gate is wired into both create and update paths. - assert "Action '" in src and "requires admin privileges" in src + from src.task_action_policy import ADMIN_ONLY_TASK_ACTIONS + + assert "run_local" in ADMIN_ONLY_TASK_ACTIONS + assert "run_script" in ADMIN_ONLY_TASK_ACTIONS + assert "ssh_command" in ADMIN_ONLY_TASK_ACTIONS def test_task_create_notification_default_allows_action_specific_defaults(): diff --git a/tests/test_task_cookbook_admin_gate.py b/tests/test_task_cookbook_admin_gate.py new file mode 100644 index 000000000..d7e72f9ef --- /dev/null +++ b/tests/test_task_cookbook_admin_gate.py @@ -0,0 +1,350 @@ +"""Task CRUD must not let non-admins schedule Cookbook serve actions.""" + +import sys +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool + +from tests.helpers.import_state import clear_fake_database_modules + +clear_fake_database_modules() + +import core.auth as core_auth +import core.database as cdb +import routes.task_routes as task_routes +from core.database import ScheduledTask +from core.database import TaskRun +from src.task_scheduler import TaskScheduler + +_REAL_DATABASE_ATTRS = { + "Base": cdb.Base, + "SessionLocal": cdb.SessionLocal, + "ScheduledTask": ScheduledTask, + "TaskRun": TaskRun, +} +if hasattr(cdb, "engine"): + _REAL_DATABASE_ATTRS["engine"] = cdb.engine + + +def _restore_module_binding(monkeypatch, name, module): + monkeypatch.setitem(sys.modules, name, module) + parent_name, _, attr = name.rpartition(".") + parent = sys.modules.get(parent_name) + if parent is not None: + monkeypatch.setattr(parent, attr, module, raising=False) + + +@pytest.fixture() +def task_db(monkeypatch, tmp_path): + _restore_module_binding(monkeypatch, "core.database", cdb) + for attr, value in _REAL_DATABASE_ATTRS.items(): + monkeypatch.setattr(cdb, attr, value, raising=False) + engine = create_engine( + f"sqlite:///{tmp_path / 'tasks.db'}", + connect_args={"check_same_thread": False}, + poolclass=NullPool, + ) + cdb.Base.metadata.create_all(engine) + testing_session = sessionmaker(bind=engine, autoflush=False, autocommit=False) + monkeypatch.setattr(task_routes, "SessionLocal", testing_session) + monkeypatch.setattr(cdb, "SessionLocal", testing_session) + return testing_session + + +@pytest.fixture() +def configured_auth(monkeypatch): + _restore_module_binding(monkeypatch, "core.auth", core_auth) + monkeypatch.setenv("AUTH_ENABLED", "true") + + class FakeAuthManager: + is_configured = True + + def is_admin(self, user): + return user == "admin" + + monkeypatch.setattr(core_auth, "AuthManager", FakeAuthManager) + + +@pytest.fixture() +def builtin_action_info(monkeypatch): + mod = sys.modules.get("src.builtin_actions") + if mod is None: + import src.builtin_actions as mod + monkeypatch.setattr( + mod, + "BUILTIN_ACTION_INFO", + { + "summarize_emails": "Summarize emails", + "cookbook_serve": "Serve Cookbook model", + }, + raising=False, + ) + + +def _req(user): + return SimpleNamespace(state=SimpleNamespace(current_user=user)) + + +def _endpoint(method, path): + router = task_routes.setup_task_routes(MagicMock()) + for route in router.routes: + if getattr(route, "path", None) == path and method in getattr(route, "methods", set()): + return route.endpoint + raise RuntimeError(f"{method} {path} not found") + + +def _cookbook_create_req(): + return task_routes.TaskCreate( + name="Serve test model", + prompt="{}", + task_type="action", + action="cookbook_serve", + trigger_type="webhook", + ) + + +def _seed_action_task( + session_factory, + task_id, + owner, + action="summarize_emails", + *, + task_type="action", + webhook_token=None, + next_run=None, +): + db = session_factory() + try: + task = ScheduledTask( + id=task_id, + owner=owner, + name=task_id, + prompt="{}", + task_type=task_type, + action=action, + trigger_type="webhook", + status="active", + output_target="session", + webhook_token=webhook_token, + next_run=next_run, + ) + db.add(task) + db.commit() + finally: + db.close() + + +@pytest.mark.asyncio +async def test_non_admin_cannot_create_cookbook_serve_task(task_db, configured_auth): + create_task = _endpoint("POST", "/api/tasks") + + with pytest.raises(HTTPException) as exc: + await create_task(_req("alice"), _cookbook_create_req()) + + assert exc.value.status_code == 403 + db = task_db() + try: + assert db.query(ScheduledTask).count() == 0 + finally: + db.close() + + +@pytest.mark.asyncio +async def test_non_admin_cannot_update_task_to_cookbook_serve(task_db, configured_auth): + _seed_action_task(task_db, "alice-task", "alice") + update_task = _endpoint("PUT", "/api/tasks/{task_id}") + + with pytest.raises(HTTPException) as exc: + await update_task( + _req("alice"), + "alice-task", + task_routes.TaskUpdate(action="cookbook_serve"), + ) + + assert exc.value.status_code == 403 + db = task_db() + try: + task = db.query(ScheduledTask).filter(ScheduledTask.id == "alice-task").first() + assert task.action == "summarize_emails" + finally: + db.close() + + +@pytest.mark.asyncio +async def test_non_admin_cannot_update_task_type_to_activate_existing_cookbook_serve( + task_db, configured_auth +): + _seed_action_task( + task_db, + "alice-task", + "alice", + action="cookbook_serve", + task_type="llm", + ) + update_task = _endpoint("PUT", "/api/tasks/{task_id}") + + with pytest.raises(HTTPException) as exc: + await update_task( + _req("alice"), + "alice-task", + task_routes.TaskUpdate(task_type="action"), + ) + + assert exc.value.status_code == 403 + db = task_db() + try: + task = db.query(ScheduledTask).filter(ScheduledTask.id == "alice-task").first() + assert task.task_type == "llm" + assert task.action == "cookbook_serve" + finally: + db.close() + + +@pytest.mark.asyncio +async def test_non_admin_cannot_manually_run_existing_cookbook_serve_task( + task_db, configured_auth +): + _seed_action_task(task_db, "alice-task", "alice", action="cookbook_serve") + scheduler = SimpleNamespace(run_task_now=MagicMock()) + router = task_routes.setup_task_routes(scheduler) + for route in router.routes: + if getattr(route, "path", None) == "/api/tasks/{task_id}/run": + run_task = route.endpoint + break + else: + raise RuntimeError("POST /api/tasks/{task_id}/run not found") + + with pytest.raises(HTTPException) as exc: + await run_task(_req("alice"), "alice-task") + + assert exc.value.status_code == 403 + scheduler.run_task_now.assert_not_called() + + +@pytest.mark.asyncio +async def test_webhook_rejects_stale_non_admin_cookbook_serve_task( + task_db, configured_auth +): + _seed_action_task( + task_db, + "alice-task", + "alice", + action="cookbook_serve", + webhook_token="secret", + ) + webhook_trigger = _endpoint("POST", "/api/tasks/{task_id}/webhook/{token}") + + with pytest.raises(HTTPException) as exc: + await webhook_trigger("alice-task", "secret") + + assert exc.value.status_code == 403 + db = task_db() + try: + task = db.query(ScheduledTask).filter(ScheduledTask.id == "alice-task").first() + assert task.status == "paused" + assert task.next_run is None + finally: + db.close() + + +@pytest.mark.asyncio +async def test_scheduler_pauses_stale_non_admin_cookbook_serve_task( + task_db, configured_auth +): + due = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=1) + _seed_action_task( + task_db, + "alice-task", + "alice", + action="cookbook_serve", + next_run=due, + ) + db = task_db() + try: + db.add(TaskRun(id="run-1", task_id="alice-task", status="queued")) + db.commit() + finally: + db.close() + + scheduler = TaskScheduler.__new__(TaskScheduler) + scheduler._task_handles = {} + await scheduler._execute_task_locked( + "alice-task", + "run-1", + gate_foreground=False, + release_executing=False, + ) + + db = task_db() + try: + task = db.query(ScheduledTask).filter(ScheduledTask.id == "alice-task").first() + run = db.query(TaskRun).filter(TaskRun.id == "run-1").first() + assert task.status == "paused" + assert task.next_run is None + assert run.status == "error" + assert run.error == "Action 'cookbook_serve' requires admin privileges" + finally: + db.close() + + +@pytest.mark.asyncio +async def test_non_admin_action_metadata_hides_cookbook_serve( + configured_auth, builtin_action_info +): + list_actions = _endpoint("GET", "/api/tasks/meta/actions") + + out = await list_actions(_req("alice")) + + action_names = {action["name"] for action in out["actions"]} + assert "cookbook_serve" not in action_names + + +@pytest.mark.asyncio +async def test_admin_can_create_cookbook_serve_task(task_db, configured_auth): + create_task = _endpoint("POST", "/api/tasks") + + out = await create_task(_req("admin"), _cookbook_create_req()) + + assert out["action"] == "cookbook_serve" + db = task_db() + try: + task = db.query(ScheduledTask).filter(ScheduledTask.id == out["id"]).first() + assert task.owner == "admin" + finally: + db.close() + + +@pytest.mark.asyncio +async def test_admin_action_metadata_includes_cookbook_serve( + configured_auth, builtin_action_info +): + list_actions = _endpoint("GET", "/api/tasks/meta/actions") + + out = await list_actions(_req("admin")) + + action_names = {action["name"] for action in out["actions"]} + assert "cookbook_serve" in action_names + + +@pytest.mark.asyncio +async def test_auth_disabled_single_user_can_create_cookbook_serve_task( + monkeypatch, task_db +): + monkeypatch.setenv("AUTH_ENABLED", "false") + create_task = _endpoint("POST", "/api/tasks") + + out = await create_task(_req(None), _cookbook_create_req()) + + assert out["action"] == "cookbook_serve" + db = task_db() + try: + task = db.query(ScheduledTask).filter(ScheduledTask.id == out["id"]).first() + assert task.owner is None + finally: + db.close()