Jake/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-07-15 12:58:04 +00:00
Code Packages Projects Releases Activity

Merge remote-tracking branch 'origin/dev'

Browse Source 
# Conflicts:
#	routes/contacts_routes.py
This commit is contained in:
pewdiepie-archdaemon
2026-07-07 00:51:34 +00:00
parent 017903de61 35f867c959
commit b5ec40d505
80 changed files with 5885 additions and 2383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routes/chat_routes.py
-2
View File
@@ -876,11 +876,9 @@ def setup_chat_routes(
# by default without having to send allow_bash in every request.
if allow_bash is not None and str(allow_bash).lower() != "true":
disabled_tools.add("bash")
_explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web")
if (
allow_web_search is not None
and str(allow_web_search).lower() != "true"
and not _explicit_web_intent
):
disabled_tools.add("web_search")
disabled_tools.add("web_fetch")
routes/contacts/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""Contacts route domain package (slice 2e, #4082/#4071).
Contains contacts_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/contacts_routes.py re-exports from here.
"""
routes/contacts/contacts_routes.py
+916
View File
@@ -0,0 +1,916 @@
"""
contacts_routes.py
CardDAV contacts integration. Reads from local Radicale, supports
search and adding new contacts.
"""
import re
import logging
import uuid
import json
import csv
import io
import os
import inspect
import httpx
from pathlib import Path
from datetime import datetime
from urllib.parse import urljoin, urlparse, urlunparse
from core.log_safety import redact_url
from fastapi import APIRouter, Query, Depends, Response, HTTPException
from typing import List, Dict, Optional
from core.middleware import require_admin
from src.url_safety import check_outbound_url
logger = logging.getLogger(__name__)
from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
DATA_DIR = Path(_DATA_DIR)
SETTINGS_FILE = Path(_SETTINGS_FILE)
LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
def _load_settings():
if SETTINGS_FILE.exists():
return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
return {}
def _save_settings(settings):
from core.atomic_io import atomic_write_json
atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
def _get_carddav_config():
import os
settings = _load_settings()
password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
if password and "carddav_password" in settings:
from src.secret_storage import decrypt
password = decrypt(password)
return {
"url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
"username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
"password": password,
}
def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
cfg = cfg or _get_carddav_config()
return bool((cfg.get("url") or "").strip())
def _validate_carddav_url(url: str) -> str:
cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
ok, reason = check_outbound_url(
cleaned,
block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
)
if not ok:
raise ValueError(f"Rejected CardDAV URL: {reason}")
return cleaned
def _carddav_base_url(cfg: Dict) -> str:
return _validate_carddav_url(cfg.get("url") or "")
def _normalize_contact(contact: Dict) -> Dict:
emails = []
for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
e = str(e or "").strip()
if e and e not in emails:
emails.append(e)
phones = []
for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
p = str(p or "").strip()
if p and p not in phones:
phones.append(p)
name = str(contact.get("name") or "").strip()
if not name and emails:
name = emails[0].split("@")[0]
address = str(contact.get("address") or "").strip()
return {
"uid": str(contact.get("uid") or uuid.uuid4()),
"name": name,
"emails": emails,
"phones": phones,
"address": address,
}
def _load_local_contacts() -> List[Dict]:
try:
if not LOCAL_CONTACTS_FILE.exists():
return []
data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
rows = data.get("contacts", data) if isinstance(data, dict) else data
return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
except Exception as e:
logger.error(f"Failed to load local contacts: {e}")
return []
def _save_local_contacts(contacts: List[Dict]) -> None:
from core.atomic_io import atomic_write_json
DATA_DIR.mkdir(parents=True, exist_ok=True)
atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
_contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
_contact_cache["fetched_at"] = datetime.utcnow()
# ── vCard parsing ──
def _vunesc(value: str) -> str:
"""Reverse _vesc() — turn escaped vCard text back into the raw value.
Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
if not value:
return value
out = []
i = 0
while i < len(value):
ch = value[i]
if ch == "\\" and i + 1 < len(value):
nxt = value[i + 1]
if nxt in ("n", "N"):
out.append("\n")
elif nxt in (",", ";", "\\"):
out.append(nxt)
else:
out.append(nxt)
i += 2
else:
out.append(ch)
i += 1
return "".join(out)
def _parse_vcards(text: str) -> List[Dict]:
"""Parse a stream of vCards into dicts with name, email, phone."""
# Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
# space or tab is a continuation of the previous logical line. Real
# CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
# PHOTO lines, and splitting on raw newlines without unfolding dropped the
# continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
# email/name.
text = re.sub(r"\r\n[ \t]", "", text or "")
text = re.sub(r"\n[ \t]", "", text)
contacts = []
for block in re.split(r"BEGIN:VCARD", text):
if not block.strip():
continue
contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
for line in block.split("\n"):
line = line.strip()
# Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
# that Apple Contacts / iCloud / many CardDAV servers emit by
# default — without this the property-name checks below miss those
# lines and silently drop the email / phone. The group token only
# precedes the property name, so it is safe to strip for matching
# and value extraction, and a no-op for non-grouped lines.
name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
if name_part.startswith("FN:") or name_part.startswith("FN;"):
contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
elif name_part.startswith("EMAIL"):
# Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
if ":" in name_part:
email_addr = _vunesc(name_part.split(":", 1)[1])
if email_addr and email_addr not in contact["emails"]:
contact["emails"].append(email_addr)
elif name_part.startswith("TEL"):
if ":" in name_part:
phone = _vunesc(name_part.split(":", 1)[1])
if phone and phone not in contact["phones"]:
contact["phones"].append(phone)
elif name_part.startswith("ADR"):
# vCard ADR is 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
# Recover a human-readable string by joining non-empty
# components with ", ".
if ":" in name_part:
raw = name_part.split(":", 1)[1]
parts = [_vunesc(p).strip() for p in raw.split(";")]
contact["address"] = ", ".join(p for p in parts if p)
elif name_part.startswith("UID:"):
contact["uid"] = _vunesc(name_part[4:])
if contact["name"] or contact["emails"]:
contacts.append(contact)
return contacts
def _vesc(value: str) -> str:
"""Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
or any value containing a newline produces a malformed vCard (broken
N/FN fields) or could inject arbitrary properties."""
return (
(value or "")
.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "")
.replace(",", "\\,")
.replace(";", "\\;")
)
def _build_vcard(name: str, email: str, uid: Optional[str] = None,
emails: Optional[List[str]] = None,
phones: Optional[List[str]] = None,
address: Optional[str] = None) -> str:
"""Build a vCard. Accepts either a single `email` (legacy callers) or
full `emails`/`phones` lists (edit path). The first email is marked
PREF=1. All values are RFC-6350-escaped."""
if not uid:
uid = str(uuid.uuid4())
# Normalize email lists — `email` arg is a convenience for single-email
# creation; `emails` (if given) is authoritative.
email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
# Try to split name into first/last
parts = name.strip().split()
if len(parts) >= 2:
first = parts[0]
last = " ".join(parts[1:])
else:
first = name
last = ""
# N field is structured (5 components separated by ';') — escape each
# component individually so a comma in the name doesn't split it.
n_field = f"{_vesc(last)};{_vesc(first)};;;"
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"UID:{_vesc(uid)}",
f"FN:{_vesc(name)}",
f"N:{n_field}",
]
for i, em in enumerate(email_list):
# First email is the preferred one.
lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
for ph in phone_list:
lines.append(f"TEL:{_vesc(ph)}")
# Address: stuff the whole human-readable string into the street
# component of ADR. vCard ADR has 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
addr = (address or "").strip()
if addr:
lines.append(f"ADR:;;{_vesc(addr)};;;;")
lines.append("END:VCARD")
return "\r\n".join(lines) + "\r\n"
# ── In-memory cache ──
_contact_cache = {"contacts": [], "fetched_at": None}
def _abs_url(href: str) -> str:
"""Combine a multistatus <href> (an absolute path like
/user/contacts/x.vcf) with the configured CardDAV server origin so we
get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
for the configured origin; a cross-origin href is treated as a path on the
configured server so a malicious CardDAV response cannot redirect later
writes/deletes to cloud metadata or another host."""
cfg = _get_carddav_config()
base = _carddav_base_url(cfg)
base_p = urlparse(base)
joined = urljoin(base.rstrip("/") + "/", href or "")
joined_p = urlparse(joined)
if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
return _validate_carddav_url(joined)
# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
# alongside the resource href. Lets us map each contact's UID to the real
# server resource path (which is NOT always <uid>.vcf for contacts created
# by other clients).
_ADDRESSBOOK_QUERY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
'<D:prop><D:getetag/><C:address-data/></D:prop>'
'<C:filter/>'
'</C:addressbook-query>'
)
def _fetch_via_report(cfg, auth):
"""Try a CardDAV REPORT addressbook-query — returns contacts WITH an
`href` field, or None if the server doesn't support it / errors."""
from defusedxml import ElementTree as ET
try:
r = httpx.request(
"REPORT", cfg["url"],
content=_ADDRESSBOOK_QUERY.encode("utf-8"),
headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
auth=auth, timeout=10,
)
if r.status_code not in (207, 200):
return None
root = ET.fromstring(r.text)
ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
out = []
for resp in root.findall("D:response", ns):
href_el = resp.find("D:href", ns)
data_el = resp.find(".//C:address-data", ns)
if href_el is None or data_el is None or not (data_el.text or "").strip():
continue
parsed = _parse_vcards(data_el.text)
if not parsed:
continue
c = parsed[0]
c["href"] = href_el.text.strip()
out.append(c)
# If the REPORT parsed to ZERO contacts, don't trust it — some
# CardDAV servers treat an empty <filter/> as "match nothing" and
# return a valid-but-empty 207. Return None so the caller falls
# back to the plain GET (which lists everything). A genuinely empty
# address book just costs one extra GET that also returns nothing.
if not out:
return None
return out
except Exception as e:
logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
return None
def _fetch_contacts(force=False):
"""Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
if not force and _contact_cache["fetched_at"]:
age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
if age < 60:
return _contact_cache["contacts"]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
try:
cfg["url"] = _carddav_base_url(cfg)
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
# Preferred path: REPORT gives us hrefs for reliable edit/delete.
contacts = _fetch_via_report(cfg, auth)
if contacts is None:
# Fallback: plain GET, concatenated vCards, no hrefs.
r = httpx.get(cfg["url"], auth=auth, timeout=10)
if r.status_code != 200:
logger.warning(f"CardDAV returned {r.status_code}")
return _contact_cache["contacts"]
contacts = _parse_vcards(r.text)
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
except Exception as e:
logger.error(f"Failed to fetch contacts: {e}")
return _contact_cache["contacts"]
def _resolve_resource_url(uid: str) -> str:
"""Map a contact UID to its real CardDAV resource URL. Uses the href
captured during fetch when available (handles contacts whose filename
!= UID); falls back to the <uid>.vcf guess for app-created contacts or
when no href is known."""
def _lookup():
for c in _contact_cache.get("contacts", []):
if c.get("uid") == uid and c.get("href"):
return _abs_url(c["href"])
return None
found = _lookup()
if found:
return found
# Not in cache (or no href) — refresh once and retry before guessing.
try:
_fetch_contacts(force=True)
except Exception:
pass
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
"""Add a new contact via CardDAV or local contacts."""
email = (email or "").strip()
phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = email.lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
return True
contacts.append(_normalize_contact({
"name": name,
"emails": [email] if email else [],
"phones": phone_list,
"address": address,
}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
try:
url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
# Invalidate cache
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to create contact: {e}")
return False
def _vcard_url(uid: str) -> str:
"""The CardDAV resource URL for a given contact UID. The uid is URL-
encoded so a value containing '/', '..' or other path chars can't
escape the collection and target an arbitrary CardDAV resource."""
from urllib.parse import quote
cfg = _get_carddav_config()
return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
def _import_vcards(text: str) -> Dict:
"""Import a (possibly multi-card) .vcf blob. Each card is PUT to the
CardDAV server PRESERVING its full original content (ADR/ORG/photo/
etc.) — we don't rebuild it, just ensure it has VERSION + UID and
normalize line endings. Returns {imported, failed, total}."""
from urllib.parse import quote
cfg = _get_carddav_config()
if not cfg.get("url"):
parsed = _parse_vcards(text)
contacts = _load_local_contacts()
existing = {
e.lower()
for c in contacts
for e in (c.get("emails") or [])
if e
}
imported = 0
for c in parsed:
emails = [e for e in (c.get("emails") or []) if e]
if emails and any(e.lower() in existing for e in emails):
continue
contacts.append(_normalize_contact(c))
for e in emails:
existing.add(e.lower())
imported += 1
if imported:
_save_local_contacts(contacts)
return {"imported": imported, "failed": 0, "total": len(parsed)}
try:
base_url = _carddav_base_url(cfg)
except ValueError as e:
logger.warning("CardDAV import URL rejected: %s", e)
return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
# Split into individual cards. re.split drops the BEGIN line, so we
# re-add it. Normalize CRLF.
raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
blocks = []
for chunk in raw.split("BEGIN:VCARD"):
chunk = chunk.strip()
if not chunk:
continue
# Trim anything after END:VCARD (defensive).
end = chunk.upper().find("END:VCARD")
body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
blocks.append("BEGIN:VCARD\n" + body)
imported = 0
failed = 0
for block in blocks:
# Extract or assign a UID.
m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
if not m:
# Inject a UID right after the VERSION line (or after BEGIN).
if re.search(r"^VERSION:", block, re.MULTILINE):
block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
else:
block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
elif not re.search(r"^VERSION:", block, re.MULTILINE):
block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
vcard = block.replace("\n", "\r\n") + "\r\n"
url = base_url + "/" + quote(uid, safe="") + ".vcf"
try:
r = httpx.put(
url, data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth, timeout=15,
)
if r.status_code in (200, 201, 204):
imported += 1
else:
failed += 1
logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
except Exception as e:
failed += 1
logger.error(f"Import PUT {uid} failed: {e}")
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": len(blocks)}
def _import_csv_contacts(text: str) -> Dict:
"""Import contacts from CSV. Supports common headers:
name/full_name/display_name, email/email_address/e-mail, phone/tel.
Falls back to first columns as name,email,phone when no headers exist."""
raw = (text or "").strip()
if not raw:
return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
try:
sample = raw[:2048]
dialect = csv.Sniffer().sniff(sample)
except Exception:
dialect = csv.excel
stream = io.StringIO(raw)
try:
has_header = csv.Sniffer().has_header(raw[:2048])
except Exception:
has_header = True
rows = []
if has_header:
reader = csv.DictReader(stream, dialect=dialect)
for row in reader:
lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
name = (
lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
or lowered.get("display name") or lowered.get("display_name")
or lowered.get("fn") or ""
)
email = (
lowered.get("email") or lowered.get("email address")
or lowered.get("email_address") or lowered.get("e-mail")
or lowered.get("mail") or ""
)
phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
rows.append((name, email, phone))
else:
stream.seek(0)
reader = csv.reader(stream, dialect=dialect)
for row in reader:
cols = [(c or "").strip() for c in row]
if not any(cols):
continue
rows.append((
cols[0] if len(cols) > 0 else "",
cols[1] if len(cols) > 1 else "",
cols[2] if len(cols) > 2 else "",
))
imported = 0
failed = 0
total = 0
existing_emails = {
e.lower()
for c in _fetch_contacts()
for e in (c.get("emails") or [])
if e
}
for name, email, phone in rows:
email = (email or "").strip()
name = (name or "").strip() or (email.split("@")[0] if email else "")
if not email:
continue
total += 1
if email.lower() in existing_emails:
continue
ok = _create_contact(name, email)
if ok:
imported += 1
existing_emails.add(email.lower())
# If the CSV had a phone number, rewrite the just-created row
# through the richer update path so phone lands in CardDAV too.
if phone:
try:
contacts = _fetch_contacts(force=True)
created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
if created and created.get("uid"):
_update_contact(created["uid"], name, [email], [phone])
except Exception:
pass
else:
failed += 1
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": total}
def _contacts_to_vcf(contacts: List[Dict]) -> str:
return "".join(
_build_vcard(
c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
"",
uid=c.get("uid") or str(uuid.uuid4()),
emails=c.get("emails") or [],
phones=c.get("phones") or [],
)
for c in contacts
)
def _contacts_to_csv(contacts: List[Dict]) -> str:
out = io.StringIO()
writer = csv.writer(out)
writer.writerow(["name", "email", "phone"])
for c in contacts:
emails = c.get("emails") or [""]
phones = c.get("phones") or [""]
max_len = max(len(emails), len(phones), 1)
for i in range(max_len):
writer.writerow([
c.get("name") or "",
emails[i] if i < len(emails) else "",
phones[i] if i < len(phones) else "",
])
return out.getvalue()
def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
"""Rewrite an existing contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
found = False
out = []
for c in contacts:
if c.get("uid") == uid:
# Preserve existing address when caller passes "" (only
# updating name/emails/phones, not touching address).
addr = address if address else c.get("address", "")
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
found = True
else:
out.append(c)
if not found:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
_save_local_contacts(out)
return True
vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
# Use the real resource href (handles externally-created contacts whose
# filename != UID); falls back to the <uid>.vcf guess.
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to update contact: {e}")
return False
def _delete_contact(uid: str) -> bool:
"""Delete a contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
remaining = [c for c in contacts if c.get("uid") != uid]
_save_local_contacts(remaining)
return True
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.delete(url, auth=auth, timeout=10)
if r.status_code in (200, 204, 404):
# Invalidate cache so the next fetch sees the server truth.
_contact_cache["fetched_at"] = None
# Verify: force a fresh fetch and check the UID is actually gone.
# A 404 on the guessed URL ({uid}.vcf) can mean the contact
# lives at a different resource URL — the DELETE missed it but
# we'd silently report success. This check catches that.
fresh = _fetch_contacts(force=True)
still_there = any(c.get("uid") == uid for c in fresh)
if still_there:
logger.warning(
f"CardDAV DELETE reported success for {uid} "
f"but UID still present after re-fetch — "
f"resource URL may differ from {redact_url(url)}"
)
return False
if r.status_code == 404:
logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
return True
logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to delete contact: {e}")
return False
# ── Routes ──
def setup_contacts_routes():
router = APIRouter(prefix="/api/contacts", tags=["contacts"])
@router.get("/list")
async def list_contacts(_admin: str = Depends(require_admin)):
"""List all contacts."""
contacts = _fetch_contacts()
return {"contacts": contacts, "count": len(contacts)}
@router.get("/search")
async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
"""Search contacts by name or email. Returns up to 10 matches."""
contacts = _fetch_contacts()
if not q:
return {"results": []}
q_lower = q.lower()
results = []
for c in contacts:
if q_lower in c["name"].lower():
results.append(c)
continue
for em in c["emails"]:
if q_lower in em.lower():
results.append(c)
break
return {"results": results[:10]}
@router.post("/add")
async def add_contact(data: dict, _admin: str = Depends(require_admin)):
"""Add a new contact."""
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
phone = (data.get("phone") or "").strip()
phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
if phone and phone not in phones:
phones.insert(0, phone)
address = (data.get("address") or "").strip()
if not name and email:
name = email.split("@")[0]
if not name and not email and not phones and not address:
return {"success": False, "error": "Name, email, phone, or address required"}
if not name:
name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
contacts = _fetch_contacts()
for c in contacts:
if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
return {"success": True, "message": "Already exists", "contact": c}
if phones and any(p in (c.get("phones") or []) for p in phones):
return {"success": True, "message": "Already exists", "contact": c}
create_params = inspect.signature(_create_contact).parameters
if "phones" in create_params:
ok = _create_contact(name, email, address, phones=phones)
elif len(create_params) >= 3:
ok = _create_contact(name, email, address)
else:
ok = _create_contact(name, email)
# If a phone was provided, do an immediate update to thread it
# through (the simple _create_contact signature only takes name +
# email + address; phones happen via update).
if ok and phones and "phones" not in create_params:
try:
fresh = _fetch_contacts(force=True)
created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
if created:
_update_contact(
created["uid"], name,
created.get("emails", []),
phones,
address,
)
except Exception:
pass
return {"success": ok}
@router.post("/import")
async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
"""Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
# Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
# in the JSON body) would otherwise reach .strip() and 500 with an
# AttributeError instead of degrading to a clean "no data" response.
text = str(data.get("vcf") or data.get("text") or "")
csv_text = str(data.get("csv") or "")
if text.strip():
if "BEGIN:VCARD" not in text.upper():
return {"success": False, "error": "No vCard data found"}
result = _import_vcards(text)
elif csv_text.strip():
result = _import_csv_contacts(csv_text)
else:
return {"success": False, "error": "No contact data found"}
result["success"] = result.get("imported", 0) > 0
return result
@router.get("/export")
async def export_contacts(
format: str = Query("vcf", pattern="^(vcf|csv)$"),
_admin: str = Depends(require_admin),
):
"""Export all contacts as vCard or CSV."""
contacts = _fetch_contacts(force=True)
if format == "csv":
content = _contacts_to_csv(contacts)
media_type = "text/csv; charset=utf-8"
filename = "odysseus-contacts.csv"
else:
content = _contacts_to_vcf(contacts)
media_type = "text/vcard; charset=utf-8"
filename = "odysseus-contacts.vcf"
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/config")
async def get_config(_admin: str = Depends(require_admin)):
cfg = _get_carddav_config()
# Mask password
if cfg["password"]:
cfg["password"] = "***"
return cfg
@router.put("/config")
async def update_config(data: dict, _admin: str = Depends(require_admin)):
settings = _load_settings()
for key in ("carddav_url", "carddav_username", "carddav_password"):
if key in data:
if key == "carddav_url" and str(data[key] or "").strip():
try:
settings[key] = _validate_carddav_url(data[key])
except ValueError as e:
raise HTTPException(400, str(e))
else:
value = data[key]
if key == "carddav_password" and value:
from src.secret_storage import encrypt
value = encrypt(value)
settings[key] = value
_save_settings(settings)
# Force re-fetch
_contact_cache["fetched_at"] = None
return {"success": True}
@router.delete("/clear")
async def clear_contacts(_admin: str = Depends(require_admin)):
"""Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
_save_local_contacts([])
return {"success": True}
# NOTE: the /{uid} routes are declared LAST so the literal paths above
# (/list, /search, /add, /config) win — otherwise PUT /config would
# match PUT /{uid} with uid="config".
@router.put("/{uid}")
async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
"""Edit an existing contact — name / emails / phones / address."""
name = (data.get("name") or "").strip()
emails = data.get("emails")
phones = data.get("phones")
if emails is None and data.get("email"):
emails = [data["email"]]
emails = [e.strip() for e in (emails or []) if e and e.strip()]
phones = [p.strip() for p in (phones or []) if p and p.strip()]
address = (data.get("address") or "").strip()
if not name and not emails and not address:
return {"success": False, "error": "Name, email, or address required"}
if not name and emails:
name = emails[0].split("@")[0]
ok = _update_contact(uid, name, emails, phones, address)
return {"success": ok}
@router.delete("/{uid}")
async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
"""Delete a contact by UID."""
if not uid:
return {"success": False, "error": "UID required"}
ok = _delete_contact(uid)
return {"success": ok}
return router
routes/contacts_routes.py
+8 -907
View File
@@ -1,912 +1,13 @@
"""
contacts_routes.py
"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py.
CardDAV contacts integration. Reads from local Radicale, supports
search and adding new contacts.
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``,
``importlib.import_module("routes.contacts_routes")``, and string-targeted
monkeypatches all operate on the same object the application actually uses.
"""
import re
import logging
import uuid
import json
import csv
import io
import os
import inspect
import httpx
from pathlib import Path
from datetime import datetime
from urllib.parse import urljoin, urlparse, urlunparse
import sys as _sys
from core.log_safety import redact_url
from fastapi import APIRouter, Query, Depends, Response, HTTPException
from typing import List, Dict, Optional
from routes.contacts import contacts_routes as _canonical # noqa: F401
from core.middleware import require_admin
from src.url_safety import check_outbound_url
logger = logging.getLogger(__name__)
from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
DATA_DIR = Path(_DATA_DIR)
SETTINGS_FILE = Path(_SETTINGS_FILE)
LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
def _load_settings():
if SETTINGS_FILE.exists():
return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
return {}
def _save_settings(settings):
from core.atomic_io import atomic_write_json
atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
def _get_carddav_config():
import os
settings = _load_settings()
password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
if password and "carddav_password" in settings:
from src.secret_storage import decrypt
password = decrypt(password)
return {
"url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
"username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
"password": password,
}
def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
cfg = cfg or _get_carddav_config()
return bool((cfg.get("url") or "").strip())
def _validate_carddav_url(url: str) -> str:
cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
ok, reason = check_outbound_url(
cleaned,
block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
)
if not ok:
raise ValueError(f"Rejected CardDAV URL: {reason}")
return cleaned
def _carddav_base_url(cfg: Dict) -> str:
return _validate_carddav_url(cfg.get("url") or "")
def _normalize_contact(contact: Dict) -> Dict:
emails = []
for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
e = str(e or "").strip()
if e and e not in emails:
emails.append(e)
phones = []
for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
p = str(p or "").strip()
if p and p not in phones:
phones.append(p)
name = str(contact.get("name") or "").strip()
if not name and emails:
name = emails[0].split("@")[0]
address = str(contact.get("address") or "").strip()
return {
"uid": str(contact.get("uid") or uuid.uuid4()),
"name": name,
"emails": emails,
"phones": phones,
"address": address,
}
def _load_local_contacts() -> List[Dict]:
try:
if not LOCAL_CONTACTS_FILE.exists():
return []
data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
rows = data.get("contacts", data) if isinstance(data, dict) else data
return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
except Exception as e:
logger.error(f"Failed to load local contacts: {e}")
return []
def _save_local_contacts(contacts: List[Dict]) -> None:
from core.atomic_io import atomic_write_json
DATA_DIR.mkdir(parents=True, exist_ok=True)
atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
_contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
_contact_cache["fetched_at"] = datetime.utcnow()
# ── vCard parsing ──
def _vunesc(value: str) -> str:
"""Reverse _vesc() — turn escaped vCard text back into the raw value.
Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
if not value:
return value
out = []
i = 0
while i < len(value):
ch = value[i]
if ch == "\\" and i + 1 < len(value):
nxt = value[i + 1]
if nxt in ("n", "N"):
out.append("\n")
elif nxt in (",", ";", "\\"):
out.append(nxt)
else:
out.append(nxt)
i += 2
else:
out.append(ch)
i += 1
return "".join(out)
def _parse_vcards(text: str) -> List[Dict]:
"""Parse a stream of vCards into dicts with name, email, phone."""
# Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
# space or tab is a continuation of the previous logical line. Real
# CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
# PHOTO lines, and splitting on raw newlines without unfolding dropped the
# continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
# email/name.
text = re.sub(r"\r\n[ \t]", "", text or "")
text = re.sub(r"\n[ \t]", "", text)
contacts = []
for block in re.split(r"BEGIN:VCARD", text):
if not block.strip():
continue
contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
for line in block.split("\n"):
line = line.strip()
# Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
# that Apple Contacts / iCloud / many CardDAV servers emit by
# default — without this the property-name checks below miss those
# lines and silently drop the email / phone. The group token only
# precedes the property name, so it is safe to strip for matching
# and value extraction, and a no-op for non-grouped lines.
name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
if name_part.startswith("FN:") or name_part.startswith("FN;"):
contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
elif name_part.startswith("EMAIL"):
# Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
if ":" in name_part:
email_addr = _vunesc(name_part.split(":", 1)[1])
if email_addr and email_addr not in contact["emails"]:
contact["emails"].append(email_addr)
elif name_part.startswith("TEL"):
if ":" in name_part:
phone = _vunesc(name_part.split(":", 1)[1])
if phone and phone not in contact["phones"]:
contact["phones"].append(phone)
elif name_part.startswith("ADR"):
# vCard ADR is 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
# Recover a human-readable string by joining non-empty
# components with ", ".
if ":" in name_part:
raw = name_part.split(":", 1)[1]
parts = [_vunesc(p).strip() for p in raw.split(";")]
contact["address"] = ", ".join(p for p in parts if p)
elif name_part.startswith("UID:"):
contact["uid"] = _vunesc(name_part[4:])
if contact["name"] or contact["emails"]:
contacts.append(contact)
return contacts
def _vesc(value: str) -> str:
"""Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
or any value containing a newline produces a malformed vCard (broken
N/FN fields) or could inject arbitrary properties."""
return (
(value or "")
.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "")
.replace(",", "\\,")
.replace(";", "\\;")
)
def _build_vcard(name: str, email: str, uid: Optional[str] = None,
emails: Optional[List[str]] = None,
phones: Optional[List[str]] = None,
address: Optional[str] = None) -> str:
"""Build a vCard. Accepts either a single `email` (legacy callers) or
full `emails`/`phones` lists (edit path). The first email is marked
PREF=1. All values are RFC-6350-escaped."""
if not uid:
uid = str(uuid.uuid4())
# Normalize email lists — `email` arg is a convenience for single-email
# creation; `emails` (if given) is authoritative.
email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
# Try to split name into first/last
parts = name.strip().split()
if len(parts) >= 2:
first = parts[0]
last = " ".join(parts[1:])
else:
first = name
last = ""
# N field is structured (5 components separated by ';') — escape each
# component individually so a comma in the name doesn't split it.
n_field = f"{_vesc(last)};{_vesc(first)};;;"
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"UID:{_vesc(uid)}",
f"FN:{_vesc(name)}",
f"N:{n_field}",
]
for i, em in enumerate(email_list):
# First email is the preferred one.
lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
for ph in phone_list:
lines.append(f"TEL:{_vesc(ph)}")
# Address: stuff the whole human-readable string into the street
# component of ADR. vCard ADR has 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
addr = (address or "").strip()
if addr:
lines.append(f"ADR:;;{_vesc(addr)};;;;")
lines.append("END:VCARD")
return "\r\n".join(lines) + "\r\n"
# ── In-memory cache ──
_contact_cache = {"contacts": [], "fetched_at": None}
def _abs_url(href: str) -> str:
"""Combine a multistatus <href> (an absolute path like
/user/contacts/x.vcf) with the configured CardDAV server origin so we
get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
for the configured origin; a cross-origin href is treated as a path on the
configured server so a malicious CardDAV response cannot redirect later
writes/deletes to cloud metadata or another host."""
cfg = _get_carddav_config()
base = _carddav_base_url(cfg)
base_p = urlparse(base)
joined = urljoin(base.rstrip("/") + "/", href or "")
joined_p = urlparse(joined)
if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
return _validate_carddav_url(joined)
# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
# alongside the resource href. Lets us map each contact's UID to the real
# server resource path (which is NOT always <uid>.vcf for contacts created
# by other clients).
_ADDRESSBOOK_QUERY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
'<D:prop><D:getetag/><C:address-data/></D:prop>'
'<C:filter/>'
'</C:addressbook-query>'
)
def _fetch_via_report(cfg, auth):
"""Try a CardDAV REPORT addressbook-query — returns contacts WITH an
`href` field, or None if the server doesn't support it / errors."""
from defusedxml import ElementTree as ET
try:
r = httpx.request(
"REPORT", cfg["url"],
content=_ADDRESSBOOK_QUERY.encode("utf-8"),
headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
auth=auth, timeout=10,
)
if r.status_code not in (207, 200):
return None
root = ET.fromstring(r.text)
ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
out = []
for resp in root.findall("D:response", ns):
href_el = resp.find("D:href", ns)
data_el = resp.find(".//C:address-data", ns)
if href_el is None or data_el is None or not (data_el.text or "").strip():
continue
parsed = _parse_vcards(data_el.text)
if not parsed:
continue
c = parsed[0]
c["href"] = href_el.text.strip()
out.append(c)
# If the REPORT parsed to ZERO contacts, don't trust it — some
# CardDAV servers treat an empty <filter/> as "match nothing" and
# return a valid-but-empty 207. Return None so the caller falls
# back to the plain GET (which lists everything). A genuinely empty
# address book just costs one extra GET that also returns nothing.
if not out:
return None
return out
except Exception as e:
logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
return None
def _fetch_contacts(force=False):
"""Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
if not force and _contact_cache["fetched_at"]:
age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
if age < 60:
return _contact_cache["contacts"]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
try:
cfg["url"] = _carddav_base_url(cfg)
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
# Preferred path: REPORT gives us hrefs for reliable edit/delete.
contacts = _fetch_via_report(cfg, auth)
if contacts is None:
# Fallback: plain GET, concatenated vCards, no hrefs.
r = httpx.get(cfg["url"], auth=auth, timeout=10)
if r.status_code != 200:
logger.warning(f"CardDAV returned {r.status_code}")
return _contact_cache["contacts"]
contacts = _parse_vcards(r.text)
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
except Exception as e:
logger.error(f"Failed to fetch contacts: {e}")
return _contact_cache["contacts"]
def _resolve_resource_url(uid: str) -> str:
"""Map a contact UID to its real CardDAV resource URL. Uses the href
captured during fetch when available (handles contacts whose filename
!= UID); falls back to the <uid>.vcf guess for app-created contacts or
when no href is known."""
def _lookup():
for c in _contact_cache.get("contacts", []):
if c.get("uid") == uid and c.get("href"):
return _abs_url(c["href"])
return None
found = _lookup()
if found:
return found
# Not in cache (or no href) — refresh once and retry before guessing.
try:
_fetch_contacts(force=True)
except Exception:
pass
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
"""Add a new contact via CardDAV or local contacts."""
email = (email or "").strip()
phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = email.lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
return True
contacts.append(_normalize_contact({"name": name, "emails": [email] if email else [], "phones": phone_list, "address": address}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
try:
url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
# Invalidate cache
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to create contact: {e}")
return False
def _vcard_url(uid: str) -> str:
"""The CardDAV resource URL for a given contact UID. The uid is URL-
encoded so a value containing '/', '..' or other path chars can't
escape the collection and target an arbitrary CardDAV resource."""
from urllib.parse import quote
cfg = _get_carddav_config()
return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
def _import_vcards(text: str) -> Dict:
"""Import a (possibly multi-card) .vcf blob. Each card is PUT to the
CardDAV server PRESERVING its full original content (ADR/ORG/photo/
etc.) — we don't rebuild it, just ensure it has VERSION + UID and
normalize line endings. Returns {imported, failed, total}."""
from urllib.parse import quote
cfg = _get_carddav_config()
if not cfg.get("url"):
parsed = _parse_vcards(text)
contacts = _load_local_contacts()
existing = {
e.lower()
for c in contacts
for e in (c.get("emails") or [])
if e
}
imported = 0
for c in parsed:
emails = [e for e in (c.get("emails") or []) if e]
if emails and any(e.lower() in existing for e in emails):
continue
contacts.append(_normalize_contact(c))
for e in emails:
existing.add(e.lower())
imported += 1
if imported:
_save_local_contacts(contacts)
return {"imported": imported, "failed": 0, "total": len(parsed)}
try:
base_url = _carddav_base_url(cfg)
except ValueError as e:
logger.warning("CardDAV import URL rejected: %s", e)
return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
# Split into individual cards. re.split drops the BEGIN line, so we
# re-add it. Normalize CRLF.
raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
blocks = []
for chunk in raw.split("BEGIN:VCARD"):
chunk = chunk.strip()
if not chunk:
continue
# Trim anything after END:VCARD (defensive).
end = chunk.upper().find("END:VCARD")
body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
blocks.append("BEGIN:VCARD\n" + body)
imported = 0
failed = 0
for block in blocks:
# Extract or assign a UID.
m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
if not m:
# Inject a UID right after the VERSION line (or after BEGIN).
if re.search(r"^VERSION:", block, re.MULTILINE):
block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
else:
block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
elif not re.search(r"^VERSION:", block, re.MULTILINE):
block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
vcard = block.replace("\n", "\r\n") + "\r\n"
url = base_url + "/" + quote(uid, safe="") + ".vcf"
try:
r = httpx.put(
url, data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth, timeout=15,
)
if r.status_code in (200, 201, 204):
imported += 1
else:
failed += 1
logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
except Exception as e:
failed += 1
logger.error(f"Import PUT {uid} failed: {e}")
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": len(blocks)}
def _import_csv_contacts(text: str) -> Dict:
"""Import contacts from CSV. Supports common headers:
name/full_name/display_name, email/email_address/e-mail, phone/tel.
Falls back to first columns as name,email,phone when no headers exist."""
raw = (text or "").strip()
if not raw:
return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
try:
sample = raw[:2048]
dialect = csv.Sniffer().sniff(sample)
except Exception:
dialect = csv.excel
stream = io.StringIO(raw)
try:
has_header = csv.Sniffer().has_header(raw[:2048])
except Exception:
has_header = True
rows = []
if has_header:
reader = csv.DictReader(stream, dialect=dialect)
for row in reader:
lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
name = (
lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
or lowered.get("display name") or lowered.get("display_name")
or lowered.get("fn") or ""
)
email = (
lowered.get("email") or lowered.get("email address")
or lowered.get("email_address") or lowered.get("e-mail")
or lowered.get("mail") or ""
)
phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
rows.append((name, email, phone))
else:
stream.seek(0)
reader = csv.reader(stream, dialect=dialect)
for row in reader:
cols = [(c or "").strip() for c in row]
if not any(cols):
continue
rows.append((
cols[0] if len(cols) > 0 else "",
cols[1] if len(cols) > 1 else "",
cols[2] if len(cols) > 2 else "",
))
imported = 0
failed = 0
total = 0
existing_emails = {
e.lower()
for c in _fetch_contacts()
for e in (c.get("emails") or [])
if e
}
for name, email, phone in rows:
email = (email or "").strip()
name = (name or "").strip() or (email.split("@")[0] if email else "")
if not email:
continue
total += 1
if email.lower() in existing_emails:
continue
ok = _create_contact(name, email)
if ok:
imported += 1
existing_emails.add(email.lower())
# If the CSV had a phone number, rewrite the just-created row
# through the richer update path so phone lands in CardDAV too.
if phone:
try:
contacts = _fetch_contacts(force=True)
created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
if created and created.get("uid"):
_update_contact(created["uid"], name, [email], [phone])
except Exception:
pass
else:
failed += 1
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": total}
def _contacts_to_vcf(contacts: List[Dict]) -> str:
return "".join(
_build_vcard(
c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
"",
uid=c.get("uid") or str(uuid.uuid4()),
emails=c.get("emails") or [],
phones=c.get("phones") or [],
)
for c in contacts
)
def _contacts_to_csv(contacts: List[Dict]) -> str:
out = io.StringIO()
writer = csv.writer(out)
writer.writerow(["name", "email", "phone"])
for c in contacts:
emails = c.get("emails") or [""]
phones = c.get("phones") or [""]
max_len = max(len(emails), len(phones), 1)
for i in range(max_len):
writer.writerow([
c.get("name") or "",
emails[i] if i < len(emails) else "",
phones[i] if i < len(phones) else "",
])
return out.getvalue()
def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
"""Rewrite an existing contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
found = False
out = []
for c in contacts:
if c.get("uid") == uid:
# Preserve existing address when caller passes "" (only
# updating name/emails/phones, not touching address).
addr = address if address else c.get("address", "")
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
found = True
else:
out.append(c)
if not found:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
_save_local_contacts(out)
return True
vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
# Use the real resource href (handles externally-created contacts whose
# filename != UID); falls back to the <uid>.vcf guess.
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to update contact: {e}")
return False
def _delete_contact(uid: str) -> bool:
"""Delete a contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
remaining = [c for c in contacts if c.get("uid") != uid]
_save_local_contacts(remaining)
return True
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.delete(url, auth=auth, timeout=10)
if r.status_code in (200, 204, 404):
# Invalidate cache so the next fetch sees the server truth.
_contact_cache["fetched_at"] = None
# Verify: force a fresh fetch and check the UID is actually gone.
# A 404 on the guessed URL ({uid}.vcf) can mean the contact
# lives at a different resource URL — the DELETE missed it but
# we'd silently report success. This check catches that.
fresh = _fetch_contacts(force=True)
still_there = any(c.get("uid") == uid for c in fresh)
if still_there:
logger.warning(
f"CardDAV DELETE reported success for {uid} "
f"but UID still present after re-fetch — "
f"resource URL may differ from {redact_url(url)}"
)
return False
if r.status_code == 404:
logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
return True
logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to delete contact: {e}")
return False
# ── Routes ──
def setup_contacts_routes():
router = APIRouter(prefix="/api/contacts", tags=["contacts"])
@router.get("/list")
async def list_contacts(_admin: str = Depends(require_admin)):
"""List all contacts."""
contacts = _fetch_contacts()
return {"contacts": contacts, "count": len(contacts)}
@router.get("/search")
async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
"""Search contacts by name or email. Returns up to 10 matches."""
contacts = _fetch_contacts()
if not q:
return {"results": []}
q_lower = q.lower()
results = []
for c in contacts:
if q_lower in c["name"].lower():
results.append(c)
continue
for em in c["emails"]:
if q_lower in em.lower():
results.append(c)
break
return {"results": results[:10]}
@router.post("/add")
async def add_contact(data: dict, _admin: str = Depends(require_admin)):
"""Add a new contact."""
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
phone = (data.get("phone") or "").strip()
phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
if phone and phone not in phones:
phones.insert(0, phone)
address = (data.get("address") or "").strip()
if not name and email:
name = email.split("@")[0]
if not name and not email and not phones and not address:
return {"success": False, "error": "Name, email, phone, or address required"}
if not name:
name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
# Check if already exists by email or phone.
contacts = _fetch_contacts()
for c in contacts:
if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
return {"success": True, "message": "Already exists", "contact": c}
if phones and any(p in (c.get("phones") or []) for p in phones):
return {"success": True, "message": "Already exists", "contact": c}
create_params = inspect.signature(_create_contact).parameters
if "phones" in create_params:
ok = _create_contact(name, email, address, phones=phones)
elif len(create_params) >= 3:
ok = _create_contact(name, email, address)
else:
ok = _create_contact(name, email)
# If a phone was provided, do an immediate update to thread it
# through (the simple _create_contact signature only takes name +
# email + address; phones happen via update).
if ok and phones and "phones" not in create_params:
try:
fresh = _fetch_contacts(force=True)
created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
if created:
_update_contact(
created["uid"], name,
created.get("emails", []),
phones,
address,
)
except Exception:
pass
return {"success": ok}
@router.post("/import")
async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
"""Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
# Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
# in the JSON body) would otherwise reach .strip() and 500 with an
# AttributeError instead of degrading to a clean "no data" response.
text = str(data.get("vcf") or data.get("text") or "")
csv_text = str(data.get("csv") or "")
if text.strip():
if "BEGIN:VCARD" not in text.upper():
return {"success": False, "error": "No vCard data found"}
result = _import_vcards(text)
elif csv_text.strip():
result = _import_csv_contacts(csv_text)
else:
return {"success": False, "error": "No contact data found"}
result["success"] = result.get("imported", 0) > 0
return result
@router.get("/export")
async def export_contacts(
format: str = Query("vcf", pattern="^(vcf|csv)$"),
_admin: str = Depends(require_admin),
):
"""Export all contacts as vCard or CSV."""
contacts = _fetch_contacts(force=True)
if format == "csv":
content = _contacts_to_csv(contacts)
media_type = "text/csv; charset=utf-8"
filename = "odysseus-contacts.csv"
else:
content = _contacts_to_vcf(contacts)
media_type = "text/vcard; charset=utf-8"
filename = "odysseus-contacts.vcf"
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/config")
async def get_config(_admin: str = Depends(require_admin)):
cfg = _get_carddav_config()
# Mask password
if cfg["password"]:
cfg["password"] = "***"
return cfg
@router.put("/config")
async def update_config(data: dict, _admin: str = Depends(require_admin)):
settings = _load_settings()
for key in ("carddav_url", "carddav_username", "carddav_password"):
if key in data:
if key == "carddav_url" and str(data[key] or "").strip():
try:
settings[key] = _validate_carddav_url(data[key])
except ValueError as e:
raise HTTPException(400, str(e))
else:
value = data[key]
if key == "carddav_password" and value:
from src.secret_storage import encrypt
value = encrypt(value)
settings[key] = value
_save_settings(settings)
# Force re-fetch
_contact_cache["fetched_at"] = None
return {"success": True}
@router.delete("/clear")
async def clear_contacts(_admin: str = Depends(require_admin)):
"""Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
_save_local_contacts([])
return {"success": True}
# NOTE: the /{uid} routes are declared LAST so the literal paths above
# (/list, /search, /add, /config) win — otherwise PUT /config would
# match PUT /{uid} with uid="config".
@router.put("/{uid}")
async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
"""Edit an existing contact — name / emails / phones / address."""
name = (data.get("name") or "").strip()
emails = data.get("emails")
phones = data.get("phones")
if emails is None and data.get("email"):
emails = [data["email"]]
emails = [e.strip() for e in (emails or []) if e and e.strip()]
phones = [p.strip() for p in (phones or []) if p and p.strip()]
address = (data.get("address") or "").strip()
if not name and not emails and not address:
return {"success": False, "error": "Name, email, or address required"}
if not name and emails:
name = emails[0].split("@")[0]
ok = _update_contact(uid, name, emails, phones, address)
return {"success": ok}
@router.delete("/{uid}")
async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
"""Delete a contact by UID."""
if not uid:
return {"success": False, "error": "UID required"}
ok = _delete_contact(uid)
return {"success": ok}
return router
_sys.modules[__name__] = _canonical
routes/cookbook_helpers.py
+12
View File
@@ -584,6 +584,18 @@ def _bash_squote(v: str) -> str:
return v.replace("'", "'\\''")
# Shown by generated runner scripts when the ollama binary is missing on the
# target host. Must stay free of backticks/$( ) and be emitted single-quoted:
# an earlier version wrapped the install one-liner in backticks inside a
# double-quoted echo, which bash executed as command substitution and ran the
# system-wide installer (including on remote SSH hosts) instead of printing
# the hint.
OLLAMA_MISSING_HINT = (
"ERROR: Ollama not found on this server. Install it from "
"https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh"
)
# Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
# Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
_SERVE_CMD_ALLOWLIST = {
routes/cookbook_routes.py
+5 -2
View File
@@ -49,7 +49,7 @@ logger = logging.getLogger(__name__)
from routes.cookbook_helpers import (
_SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
_validate_local_dir, _validate_gpus, _shell_path,
_ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase,
_ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
_safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
_append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
load_stored_hf_token,
@@ -2223,7 +2223,10 @@ def setup_cookbook_routes() -> APIRouter:
runner_lines.append(' exec 3<&-; exec 3>&-')
runner_lines.append('done')
runner_lines.append('if ! command -v ollama &>/dev/null; then')
runner_lines.append(' echo "ERROR: Ollama not found on this server. Install it from https://ollama.com/download or `curl -fsSL https://ollama.com/install.sh | sh`."')
# Single-quoted on purpose: backticks inside a double-quoted
# echo are command substitution, and this line used to run the
# curl|sh installer on the target host instead of printing it.
runner_lines.append(f" echo '{_bash_squote(OLLAMA_MISSING_HINT)}'")
runner_lines.append(' echo')
runner_lines.append(' echo "=== Process exited with code 127 ==="')
runner_lines.append(' exec bash -i')
routes/email_helpers.py
+32 -6
View File
@@ -349,7 +349,7 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
row = db.query(_EA).filter(_EA.id == account_id).first()
if row is None:
raise HTTPException(404, "Account not found")
if row.owner and row.owner != owner:
if not _account_visible_to_owner(row, owner):
# Treat as 404 (not 403) so we don't leak existence.
raise HTTPException(404, "Account not found")
finally:
@@ -362,6 +362,26 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
logger.error(f"Account-owner check failed: {e}")
raise HTTPException(503, "Account check failed")
def _account_visible_to_owner(row, owner: str) -> bool:
"""Whether an authenticated `owner` may act on this EmailAccount row.
Mirrors the SQL predicate in `_get_email_config`'s
`_owner_or_matching_legacy_account`: a caller sees an account they own, or a
legacy owner-less account (owner NULL/"") only when its own mailbox
(`imap_user` / `from_address`) is the caller's. `email_accounts` is the one
owner-scoped table deliberately left out of the legacy-owner migration
backfill, so ownerless rows persist on multi-user deploys — making this the
gate that keeps one tenant off another's imported mailbox and its decrypted
IMAP/SMTP credentials."""
row_owner = getattr(row, "owner", None) or ""
if row_owner:
return row_owner == owner
return owner in {
getattr(row, "imap_user", None) or "",
getattr(row, "from_address", None) or "",
}
def _q(name: str) -> str:
"""Quote an IMAP mailbox name. Defensive: escapes `\\` and `"` and wraps
in double quotes so user-supplied folder names with spaces or quotes can't
@@ -903,12 +923,13 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
try:
if account_id:
row = db.query(_EA).filter(_EA.id == account_id, _EA.enabled == True).first() # noqa: E712
# If the resolved row belongs to a different owner, treat as
# If the resolved row isn't visible to this owner, treat as
# not-found rather than silently serving it. This is a defense
# in depth — `require_owner` already calls `_assert_owns_account`
# for query-param account_ids, but other callers (cookbook
# rules, scheduled poller) may not.
if row is not None and owner and row.owner and row.owner != owner:
# rules, scheduled poller) may not. Ownerless legacy rows are
# only visible on a mailbox match, same as the fallback below.
if row is not None and owner and not _account_visible_to_owner(row, owner):
row = None
# Fallback path — restrict to this owner's accounts so we don't
# leak another user's default mailbox to an unconfigured user.
@@ -1273,10 +1294,15 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str
try:
c = _imap_connect(account_id, owner=owner)
c.select(_q(src))
status, _ = c.copy(uid, _q(dest))
# Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy()
# and store() operate on message SEQUENCE NUMBERS, so addressing them
# with a UID moved/deleted the wrong message (or silently no-oped when
# the UID exceeded the message count). Use the UID commands, matching
# the move/delete path in email_routes.py.
status, _ = c.uid("COPY", uid, _q(dest))
if status != "OK":
return False
c.store(uid, "+FLAGS", "\\Deleted")
c.uid("STORE", uid, "+FLAGS", "\\Deleted")
c.expunge()
return True
except Exception as e:
routes/history/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""History route domain package (slice 2d, #4082/#4071).
Contains history_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/history_routes.py re-exports from here.
"""
routes/history/history_routes.py
+768
View File
@@ -0,0 +1,768 @@
"""History routes — session history, truncation, fork, conversation topics."""
import json
import uuid
import logging
import re
from typing import Dict, Any, Optional
from fastapi import APIRouter, Request, HTTPException
from core.models import ChatMessage
from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
from src.topic_analyzer import analyze_topics
from routes.session_routes import (
_message_role,
_message_text,
_reject_compact_during_active_run,
_verify_session_owner,
)
logger = logging.getLogger(__name__)
_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
def _history_display_content(content: Any) -> Any:
"""Return a lightweight browser-display copy of stored message content.
Older multimodal user messages may be persisted as a JSON *string*
containing image_url blocks with inline base64 image bytes. Those bytes are
needed for model calls when the turn is first sent, but they should not be
sent back through /api/history every time the user opens the chat. The
attachment metadata already carries file ids/names for the UI cards.
"""
if isinstance(content, list):
text_parts = []
omitted_media = 0
for block in content:
if not isinstance(block, dict):
continue
if block.get("type") == "text":
text = block.get("text")
if isinstance(text, str) and text:
text_parts.append(text)
elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
omitted_media += 1
text = "\n".join(text_parts).strip()
if omitted_media and not text:
return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
return text
if not isinstance(content, str):
return content
if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
return content
stripped = content.lstrip()
if stripped.startswith("["):
try:
blocks = json.loads(content)
except (json.JSONDecodeError, TypeError, ValueError):
blocks = None
if isinstance(blocks, list):
text_parts = []
for block in blocks:
if not isinstance(block, dict):
continue
if block.get("type") == "text":
text = block.get("text")
if isinstance(text, str) and text:
text_parts.append(text)
if text_parts:
return "\n".join(text_parts).strip()
if "data:image/" in content:
return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
return content
def _merge_continue_rows_to_delete(db_messages, db1, db2):
"""DB rows to delete when merging the last two assistant messages.
Always the second assistant message (db2), plus ONLY the single
intervening "continue" user message (the one carrying "previous response
was interrupted") — matching the in-memory merge. The previous code
deleted the whole index range between the two assistant rows, destroying
any tool/system/user messages in between and desyncing the DB from the
in-memory history.
"""
to_delete = [db2]
i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
if i1 is not None and i2 is not None and i2 - 1 > i1:
between = db_messages[i2 - 1]
if getattr(between, "role", "") == "user" and "previous response was interrupted" in (getattr(between, "content", "") or ""):
to_delete.append(between)
return to_delete
def setup_history_routes(session_manager) -> APIRouter:
router = APIRouter(tags=["history"])
def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
entry = {"role": m.role, "content": _history_display_content(m.content)}
meta = {}
if m.meta_data:
try:
meta = json.loads(m.meta_data) or {}
except (json.JSONDecodeError, ValueError):
meta = {}
if m.timestamp and "timestamp" not in meta:
meta["timestamp"] = m.timestamp.isoformat() + "Z"
if meta:
entry["metadata"] = meta
return entry
@router.get("/api/history/{session_id}")
async def get_session_history(
request: Request,
session_id: str,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Dict[str, Any]:
_verify_session_owner(request, session_id)
if limit is not None:
page_limit = max(1, min(int(limit), 100))
db = SessionLocal()
try:
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session is None:
raise HTTPException(404, f"Session '{session_id}' not found")
total = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.count()
)
page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
page_offset = max(0, min(page_offset, total))
rows = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.offset(page_offset)
.limit(page_limit)
.all()
)
history_dict = [
entry for entry in (_db_history_entry(m) for m in rows)
if not (entry.get("metadata") or {}).get("hidden")
]
return {
"history": history_dict,
"model": db_session.model,
"endpoint_url": db_session.endpoint_url,
"name": db_session.name,
"offset": page_offset,
"limit": page_limit,
"total": total,
"has_more_before": page_offset > 0,
"has_more_after": page_offset + len(rows) < total,
}
finally:
db.close()
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, f"Session '{session_id}' not found")
history_dict = []
for msg in session.history:
if isinstance(msg, ChatMessage):
# Skip hidden messages (e.g. compaction summaries for AI context)
if msg.metadata and msg.metadata.get("hidden"):
continue
entry = {"role": msg.role, "content": _history_display_content(msg.content)}
if msg.metadata:
entry["metadata"] = msg.metadata
history_dict.append(entry)
elif isinstance(msg, dict):
if msg.get("metadata", {}).get("hidden"):
continue
entry = {
"role": msg.get("role", ""),
"content": _history_display_content(msg.get("content", "")),
}
if msg.get("metadata"):
entry["metadata"] = msg["metadata"]
history_dict.append(entry)
# Fallback: load from DB if in-memory is empty
if not history_dict:
db = SessionLocal()
try:
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
db_history = []
for m in db_messages:
db_history.append(_db_history_entry(m))
if db_history:
# Rebuild in-memory history from the full set so hidden
# messages (e.g. compaction summaries) are kept for AI context.
session.history = [
ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
for m in db_history
]
# Response excludes hidden messages, matching the in-memory path.
history_dict = [
m for m in db_history
if not (m.get("metadata") or {}).get("hidden")
]
except Exception as e:
logger.error(f"DB fallback failed for {session_id}: {e}")
finally:
db.close()
return {
"history": history_dict,
"model": session.model,
"endpoint_url": session.endpoint_url,
"name": session.name,
}
@router.post("/api/session/{session_id}/truncate")
async def truncate_session(request: Request, session_id: str):
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
result = session_manager.truncate_messages(session_id, keep_count)
return {"status": "ok", "kept": keep_count, "truncated": result}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Truncate error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/message")
async def add_message(request: Request, session_id: str):
"""Add a message to a session (for slash command persistence)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
role = body.get("role", "assistant")
content = body.get("content", "")
if not content:
raise HTTPException(400, "content is required")
msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
session_manager.add_message(session_id, msg)
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
@router.post("/api/session/{session_id}/delete-messages")
async def delete_messages(request: Request, session_id: str):
"""Delete specific messages by DB ID (or legacy index)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_ids = body.get("msg_ids", [])
indices = body.get("indices") # legacy fallback
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
if msg_ids:
# New ID-based delete
deleted = 0
for mid in msg_ids:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == mid,
DbChatMessage.session_id == session_id,
).first()
if db_msg:
db.delete(db_msg)
deleted += 1
# Remove from in-memory history by matching _db_id
def _get_db_id(m):
meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
return meta.get('_db_id') if isinstance(meta, dict) else None
session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
elif indices:
# Legacy index-based delete
indices = sorted(indices, reverse=True)
db_messages = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
deleted = 0
for idx in indices:
if 0 <= idx < len(db_messages):
db.delete(db_messages[idx])
deleted += 1
if 0 <= idx < len(session.history):
session.history.pop(idx)
else:
return {"status": "ok", "deleted": 0}
session.message_count = len(session.history)
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
from datetime import datetime, timezone
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
return {"status": "ok", "deleted": deleted}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Delete messages error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/edit-message")
async def edit_message(request: Request, session_id: str):
"""Edit the content of a message by its database ID."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_id = body.get("msg_id")
content = body.get("content")
if not msg_id or content is None:
raise HTTPException(400, "msg_id and content are required")
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == msg_id,
DbChatMessage.session_id == session_id,
).first()
if not db_msg:
raise HTTPException(404, "Message not found")
db_msg.content = content
meta = {}
if db_msg.meta_data:
try: meta = json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta['edited'] = True
db_msg.meta_data = json.dumps(meta)
# Update in-memory history by matching _db_id
for hmsg in session.history:
hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
if isinstance(hmsg, ChatMessage):
hmsg.content = content
hmsg.metadata['edited'] = True
elif isinstance(hmsg, dict):
hmsg['content'] = content
hmsg['metadata']['edited'] = True
break
db.commit()
return {"status": "ok"}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except HTTPException:
raise
except Exception as e:
logger.error(f"Edit message error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/mark-stopped")
async def mark_stopped(request: Request, session_id: str):
"""Mark the last assistant message as stopped by user."""
_verify_session_owner(request, session_id)
try:
session = session_manager.get_session(session_id)
# Find last assistant message and add stopped metadata
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata['stopped'] = True
if not msg.metadata.get('model'):
msg.metadata['model'] = session.model
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata']['stopped'] = True
if not msg['metadata'].get('model'):
msg['metadata']['model'] = session.model
break
# Also update in DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_messages:
meta = {}
if db_messages.meta_data:
try:
meta = _json.loads(db_messages.meta_data)
except (json.JSONDecodeError, ValueError):
pass
meta['stopped'] = True
if not meta.get('model'):
meta['model'] = session.model
db_messages.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Mark stopped error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/update-last-meta")
async def update_last_meta(request: Request, session_id: str):
"""Merge metadata into the last assistant message (e.g. save variants)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
meta_update = body.get("metadata", {})
session = session_manager.get_session(session_id)
# Update in-memory
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata.update(meta_update)
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata'].update(meta_update)
break
# Update in DB
db = SessionLocal()
try:
import json as _json
db_msg = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_msg:
meta = {}
if db_msg.meta_data:
try: meta = _json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta.update(meta_update)
db_msg.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Update last meta error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/merge-last-assistant")
async def merge_last_assistant(request: Request, session_id: str):
"""Merge the last two assistant messages into one (for continue)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
separator = body.get("separator", "\n\n")
session = session_manager.get_session(session_id)
# Find last two assistant messages in-memory
ai_indices = []
for i, msg in enumerate(session.history):
role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
if role == 'assistant':
ai_indices.append(i)
if len(ai_indices) < 2:
return {"status": "ok", "merged": False}
idx1, idx2 = ai_indices[-2], ai_indices[-1]
msg1, msg2 = session.history[idx1], session.history[idx2]
content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
merged_content = content1 + separator + content2
# Merge metadata
meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
merged_meta = {**meta1, **meta2}
merged_meta.pop('stopped', None) # no longer stopped after continue
# Update first message, remove second
if isinstance(msg1, ChatMessage):
msg1.content = merged_content
msg1.metadata = merged_meta
else:
msg1['content'] = merged_content
msg1['metadata'] = merged_meta
# Also remove the hidden "continue" user message between them if present
# It's the message at idx2-1 if it's a user message with continue text
remove_indices = [idx2]
if idx2 - 1 > idx1:
between = session.history[idx2 - 1]
between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
if between_role == 'user' and 'previous response was interrupted' in between_content:
remove_indices.insert(0, idx2 - 1)
for ri in sorted(remove_indices, reverse=True):
session.history.pop(ri)
# Update DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
# Find last two assistant messages in DB
ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
if len(ai_db) >= 2:
(_, db1), (_, db2) = ai_db[-2], ai_db[-1]
db1.content = merged_content
db1.meta_data = _json.dumps(merged_meta)
# Mirror the in-memory deletion: remove the second assistant
# message and ONLY the "continue" user message between them
# (not arbitrary tool/system/user rows). The old
# range-delete destroyed every row between the two assistant
# messages, desyncing the DB from the in-memory history.
for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
db.delete(_row)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok", "merged": True}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Merge assistant error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/fork")
async def fork_session(request: Request, session_id: str):
"""Create a new session with messages copied up to keep_count."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
# Get the source session
source = session_manager.sessions.get(session_id)
if not source:
raise HTTPException(404, "Session not found")
# Create new session
new_id = str(uuid.uuid4())
fork_name = f"\u2ADD {source.name}"
new_session = session_manager.create_session(
session_id=new_id,
name=fork_name,
endpoint_url=source.endpoint_url,
model=source.model,
rag=False,
owner=getattr(source, 'owner', None),
)
# Copy messages up to keep_count
msgs_to_copy = source.history[:keep_count]
for msg in msgs_to_copy:
# Copy the metadata dict. Sharing it would let the fork's
# persistence (add_message -> _persist_message stamps
# _db_id/timestamp onto the dict) mutate the SOURCE session's
# in-memory messages, corrupting their _db_id and breaking
# edit/delete-by-id on the original conversation.
meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
new_session.add_message(ChatMessage(msg.role, msg.content, meta))
try:
from src.event_bus import fire_event
fire_event("session_created", getattr(source, 'owner', None))
except Exception:
logger.debug("session_created event dispatch failed", exc_info=True)
return {
"status": "ok",
"id": new_id,
"name": fork_name,
"kept": len(msgs_to_copy),
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Fork error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.get("/api/conversations/topics")
async def get_conversation_topics(request: Request) -> Dict[str, Any]:
from src.auth_helpers import require_user
user = require_user(request)
try:
return analyze_topics(session_manager, owner=user or None)
except Exception as e:
raise HTTPException(500, f"Topic analysis failed: {e}")
@router.post("/api/session/{session_id}/compact")
async def compact_session(request: Request, session_id: str):
"""Manually trigger context compaction for a session."""
_verify_session_owner(request, session_id)
from src.auth_helpers import effective_user
owner = effective_user(request)
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, "Session not found")
_reject_compact_during_active_run(session_id)
try:
from src.model_context import estimate_tokens, get_context_length
from src.llm_core import llm_call_async
from src.endpoint_resolver import resolve_endpoint
if len(session.history) < 6:
return {"status": "ok", "message": "Not enough messages to compact"}
ctx_len = get_context_length(session.endpoint_url, session.model)
messages_before = session.get_context_messages()
used_before = estimate_tokens(messages_before)
pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
msg_count_before = len(session.history)
# Keep only last 4 messages, summarize the rest
keep_count = 4
older = session.history[:-keep_count]
recent = session.history[-keep_count:]
# Build text to summarize
convo_text = "\n".join(
f"{_message_role(m).upper()}: "
f"{_message_text(m)[:2000]}"
for m in older
)
# Use utility model if available
util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
compact_url = util_url or session.endpoint_url
compact_model = util_model or session.model
compact_headers = util_headers if util_url else session.headers
from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
summary = await llm_call_async(
compact_url, compact_model,
[
{"role": "system", "content": sys_prompt},
{"role": "user", "content": convo_text},
],
temperature=0.2, max_tokens=1024,
headers=compact_headers, timeout=30,
)
# Replace session history: summary as system message + recent messages
# System message holds the full summary for AI context
system_summary = ChatMessage(
role="system",
content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
metadata={"compacted": True, "hidden": True},
)
# Visible assistant message just shows stats
summary_msg = ChatMessage(
role="assistant",
content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
metadata={"compacted": True, "messages_removed": len(older)},
)
new_history = [system_summary, summary_msg] + list(recent)
session.history = new_history
session.message_count = len(session.history)
logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
# Update DB: delete old messages, insert summary
db = SessionLocal()
try:
db_msgs = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
# Delete all but the last keep_count
for m in db_msgs[:-keep_count]:
db.delete(m)
# Insert system summary (hidden, for AI context) and visible summary
import json as _json
import uuid
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
db_sys_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="system",
content=system_summary.content,
meta_data=_json.dumps(system_summary.metadata),
timestamp=now,
)
db.add(db_sys_summary)
db_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="assistant",
content=summary_msg.content,
meta_data=_json.dumps(summary_msg.metadata),
timestamp=now,
)
db.add(db_summary)
# Update session record
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
finally:
db.close()
session_manager.save_sessions()
used_after = estimate_tokens(session.get_context_messages())
pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
return {
"status": "ok",
"message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
"before": pct_before,
"after": pct_after,
}
except Exception as e:
logger.error(f"Manual compact error {session_id}: {e}")
raise HTTPException(500, str(e))
return router
routes/history_routes.py
+13 -764
View File
@@ -1,768 +1,17 @@
"""History routes — session history, truncation, fork, conversation topics."""
"""Backward-compat shim — canonical location is routes/history/history_routes.py.
import json
import uuid
import logging
import re
from typing import Dict, Any, Optional
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.history_routes``, ``from routes.history_routes import X``,
``importlib.import_module("routes.history_routes")``, and the
``import ... as history_routes`` + ``monkeypatch.setattr(history_routes, ...)``
pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py
all operate on the *same* object the application actually uses. Keeps existing
import paths working after slice 2d (#4082/#4071). Source-introspection tests
read the canonical file by path.
"""
from fastapi import APIRouter, Request, HTTPException
import sys as _sys
from core.models import ChatMessage
from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
from src.topic_analyzer import analyze_topics
from routes.session_routes import (
_message_role,
_message_text,
_reject_compact_during_active_run,
_verify_session_owner,
)
from routes.history import history_routes as _canonical # noqa: F401
logger = logging.getLogger(__name__)
_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
def _history_display_content(content: Any) -> Any:
"""Return a lightweight browser-display copy of stored message content.
Older multimodal user messages may be persisted as a JSON *string*
containing image_url blocks with inline base64 image bytes. Those bytes are
needed for model calls when the turn is first sent, but they should not be
sent back through /api/history every time the user opens the chat. The
attachment metadata already carries file ids/names for the UI cards.
"""
if isinstance(content, list):
text_parts = []
omitted_media = 0
for block in content:
if not isinstance(block, dict):
continue
if block.get("type") == "text":
text = block.get("text")
if isinstance(text, str) and text:
text_parts.append(text)
elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}:
omitted_media += 1
text = "\n".join(text_parts).strip()
if omitted_media and not text:
return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]"
return text
if not isinstance(content, str):
return content
if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content:
return content
stripped = content.lstrip()
if stripped.startswith("["):
try:
blocks = json.loads(content)
except (json.JSONDecodeError, TypeError, ValueError):
blocks = None
if isinstance(blocks, list):
text_parts = []
for block in blocks:
if not isinstance(block, dict):
continue
if block.get("type") == "text":
text = block.get("text")
if isinstance(text, str) and text:
text_parts.append(text)
if text_parts:
return "\n".join(text_parts).strip()
if "data:image/" in content:
return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content)
return content
def _merge_continue_rows_to_delete(db_messages, db1, db2):
"""DB rows to delete when merging the last two assistant messages.
Always the second assistant message (db2), plus ONLY the single
intervening "continue" user message (the one carrying "previous response
was interrupted") — matching the in-memory merge. The previous code
deleted the whole index range between the two assistant rows, destroying
any tool/system/user messages in between and desyncing the DB from the
in-memory history.
"""
to_delete = [db2]
i1 = next((i for i, m in enumerate(db_messages) if m is db1), None)
i2 = next((i for i, m in enumerate(db_messages) if m is db2), None)
if i1 is not None and i2 is not None and i2 - 1 > i1:
between = db_messages[i2 - 1]
if getattr(between, "role", "") == "user" and "previous response was interrupted" in (getattr(between, "content", "") or ""):
to_delete.append(between)
return to_delete
def setup_history_routes(session_manager) -> APIRouter:
router = APIRouter(tags=["history"])
def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
entry = {"role": m.role, "content": _history_display_content(m.content)}
meta = {}
if m.meta_data:
try:
meta = json.loads(m.meta_data) or {}
except (json.JSONDecodeError, ValueError):
meta = {}
if m.timestamp and "timestamp" not in meta:
meta["timestamp"] = m.timestamp.isoformat() + "Z"
if meta:
entry["metadata"] = meta
return entry
@router.get("/api/history/{session_id}")
async def get_session_history(
request: Request,
session_id: str,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Dict[str, Any]:
_verify_session_owner(request, session_id)
if limit is not None:
page_limit = max(1, min(int(limit), 100))
db = SessionLocal()
try:
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session is None:
raise HTTPException(404, f"Session '{session_id}' not found")
total = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.count()
)
page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
page_offset = max(0, min(page_offset, total))
rows = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.offset(page_offset)
.limit(page_limit)
.all()
)
history_dict = [
entry for entry in (_db_history_entry(m) for m in rows)
if not (entry.get("metadata") or {}).get("hidden")
]
return {
"history": history_dict,
"model": db_session.model,
"endpoint_url": db_session.endpoint_url,
"name": db_session.name,
"offset": page_offset,
"limit": page_limit,
"total": total,
"has_more_before": page_offset > 0,
"has_more_after": page_offset + len(rows) < total,
}
finally:
db.close()
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, f"Session '{session_id}' not found")
history_dict = []
for msg in session.history:
if isinstance(msg, ChatMessage):
# Skip hidden messages (e.g. compaction summaries for AI context)
if msg.metadata and msg.metadata.get("hidden"):
continue
entry = {"role": msg.role, "content": _history_display_content(msg.content)}
if msg.metadata:
entry["metadata"] = msg.metadata
history_dict.append(entry)
elif isinstance(msg, dict):
if msg.get("metadata", {}).get("hidden"):
continue
entry = {
"role": msg.get("role", ""),
"content": _history_display_content(msg.get("content", "")),
}
if msg.get("metadata"):
entry["metadata"] = msg["metadata"]
history_dict.append(entry)
# Fallback: load from DB if in-memory is empty
if not history_dict:
db = SessionLocal()
try:
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
db_history = []
for m in db_messages:
db_history.append(_db_history_entry(m))
if db_history:
# Rebuild in-memory history from the full set so hidden
# messages (e.g. compaction summaries) are kept for AI context.
session.history = [
ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
for m in db_history
]
# Response excludes hidden messages, matching the in-memory path.
history_dict = [
m for m in db_history
if not (m.get("metadata") or {}).get("hidden")
]
except Exception as e:
logger.error(f"DB fallback failed for {session_id}: {e}")
finally:
db.close()
return {
"history": history_dict,
"model": session.model,
"endpoint_url": session.endpoint_url,
"name": session.name,
}
@router.post("/api/session/{session_id}/truncate")
async def truncate_session(request: Request, session_id: str):
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
result = session_manager.truncate_messages(session_id, keep_count)
return {"status": "ok", "kept": keep_count, "truncated": result}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Truncate error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/message")
async def add_message(request: Request, session_id: str):
"""Add a message to a session (for slash command persistence)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
role = body.get("role", "assistant")
content = body.get("content", "")
if not content:
raise HTTPException(400, "content is required")
msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
session_manager.add_message(session_id, msg)
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
@router.post("/api/session/{session_id}/delete-messages")
async def delete_messages(request: Request, session_id: str):
"""Delete specific messages by DB ID (or legacy index)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_ids = body.get("msg_ids", [])
indices = body.get("indices") # legacy fallback
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
if msg_ids:
# New ID-based delete
deleted = 0
for mid in msg_ids:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == mid,
DbChatMessage.session_id == session_id,
).first()
if db_msg:
db.delete(db_msg)
deleted += 1
# Remove from in-memory history by matching _db_id
def _get_db_id(m):
meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
return meta.get('_db_id') if isinstance(meta, dict) else None
session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
elif indices:
# Legacy index-based delete
indices = sorted(indices, reverse=True)
db_messages = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
deleted = 0
for idx in indices:
if 0 <= idx < len(db_messages):
db.delete(db_messages[idx])
deleted += 1
if 0 <= idx < len(session.history):
session.history.pop(idx)
else:
return {"status": "ok", "deleted": 0}
session.message_count = len(session.history)
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
from datetime import datetime, timezone
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
return {"status": "ok", "deleted": deleted}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Delete messages error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/edit-message")
async def edit_message(request: Request, session_id: str):
"""Edit the content of a message by its database ID."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
msg_id = body.get("msg_id")
content = body.get("content")
if not msg_id or content is None:
raise HTTPException(400, "msg_id and content are required")
session = session_manager.get_session(session_id)
db = SessionLocal()
try:
db_msg = db.query(DbChatMessage).filter(
DbChatMessage.id == msg_id,
DbChatMessage.session_id == session_id,
).first()
if not db_msg:
raise HTTPException(404, "Message not found")
db_msg.content = content
meta = {}
if db_msg.meta_data:
try: meta = json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta['edited'] = True
db_msg.meta_data = json.dumps(meta)
# Update in-memory history by matching _db_id
for hmsg in session.history:
hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
if isinstance(hmsg, ChatMessage):
hmsg.content = content
hmsg.metadata['edited'] = True
elif isinstance(hmsg, dict):
hmsg['content'] = content
hmsg['metadata']['edited'] = True
break
db.commit()
return {"status": "ok"}
finally:
db.close()
except KeyError:
raise HTTPException(404, "Session not found")
except HTTPException:
raise
except Exception as e:
logger.error(f"Edit message error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/mark-stopped")
async def mark_stopped(request: Request, session_id: str):
"""Mark the last assistant message as stopped by user."""
_verify_session_owner(request, session_id)
try:
session = session_manager.get_session(session_id)
# Find last assistant message and add stopped metadata
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata['stopped'] = True
if not msg.metadata.get('model'):
msg.metadata['model'] = session.model
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata']['stopped'] = True
if not msg['metadata'].get('model'):
msg['metadata']['model'] = session.model
break
# Also update in DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_messages:
meta = {}
if db_messages.meta_data:
try:
meta = _json.loads(db_messages.meta_data)
except (json.JSONDecodeError, ValueError):
pass
meta['stopped'] = True
if not meta.get('model'):
meta['model'] = session.model
db_messages.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Mark stopped error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/update-last-meta")
async def update_last_meta(request: Request, session_id: str):
"""Merge metadata into the last assistant message (e.g. save variants)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
meta_update = body.get("metadata", {})
session = session_manager.get_session(session_id)
# Update in-memory
for msg in reversed(session.history):
if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
(isinstance(msg, dict) and msg.get('role') == 'assistant'):
if isinstance(msg, ChatMessage):
if not msg.metadata:
msg.metadata = {}
msg.metadata.update(meta_update)
else:
if 'metadata' not in msg:
msg['metadata'] = {}
msg['metadata'].update(meta_update)
break
# Update in DB
db = SessionLocal()
try:
import json as _json
db_msg = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
.order_by(DbChatMessage.timestamp.desc())
.first()
)
if db_msg:
meta = {}
if db_msg.meta_data:
try: meta = _json.loads(db_msg.meta_data)
except (json.JSONDecodeError, ValueError): pass
meta.update(meta_update)
db_msg.meta_data = _json.dumps(meta)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok"}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Update last meta error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/merge-last-assistant")
async def merge_last_assistant(request: Request, session_id: str):
"""Merge the last two assistant messages into one (for continue)."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
separator = body.get("separator", "\n\n")
session = session_manager.get_session(session_id)
# Find last two assistant messages in-memory
ai_indices = []
for i, msg in enumerate(session.history):
role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
if role == 'assistant':
ai_indices.append(i)
if len(ai_indices) < 2:
return {"status": "ok", "merged": False}
idx1, idx2 = ai_indices[-2], ai_indices[-1]
msg1, msg2 = session.history[idx1], session.history[idx2]
content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
merged_content = content1 + separator + content2
# Merge metadata
meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
merged_meta = {**meta1, **meta2}
merged_meta.pop('stopped', None) # no longer stopped after continue
# Update first message, remove second
if isinstance(msg1, ChatMessage):
msg1.content = merged_content
msg1.metadata = merged_meta
else:
msg1['content'] = merged_content
msg1['metadata'] = merged_meta
# Also remove the hidden "continue" user message between them if present
# It's the message at idx2-1 if it's a user message with continue text
remove_indices = [idx2]
if idx2 - 1 > idx1:
between = session.history[idx2 - 1]
between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
if between_role == 'user' and 'previous response was interrupted' in between_content:
remove_indices.insert(0, idx2 - 1)
for ri in sorted(remove_indices, reverse=True):
session.history.pop(ri)
# Update DB
db = SessionLocal()
try:
import json as _json
db_messages = (
db.query(DbChatMessage)
.filter(DbChatMessage.session_id == session_id)
.order_by(DbChatMessage.timestamp)
.all()
)
# Find last two assistant messages in DB
ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
if len(ai_db) >= 2:
(_, db1), (_, db2) = ai_db[-2], ai_db[-1]
db1.content = merged_content
db1.meta_data = _json.dumps(merged_meta)
# Mirror the in-memory deletion: remove the second assistant
# message and ONLY the "continue" user message between them
# (not arbitrary tool/system/user rows). The old
# range-delete destroyed every row between the two assistant
# messages, desyncing the DB from the in-memory history.
for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
db.delete(_row)
db.commit()
finally:
db.close()
session_manager.save_sessions()
return {"status": "ok", "merged": True}
except KeyError:
raise HTTPException(404, "Session not found")
except Exception as e:
logger.error(f"Merge assistant error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.post("/api/session/{session_id}/fork")
async def fork_session(request: Request, session_id: str):
"""Create a new session with messages copied up to keep_count."""
_verify_session_owner(request, session_id)
try:
body = await request.json()
keep_count = body.get("keep_count", 0)
# Get the source session
source = session_manager.sessions.get(session_id)
if not source:
raise HTTPException(404, "Session not found")
# Create new session
new_id = str(uuid.uuid4())
fork_name = f"\u2ADD {source.name}"
new_session = session_manager.create_session(
session_id=new_id,
name=fork_name,
endpoint_url=source.endpoint_url,
model=source.model,
rag=False,
owner=getattr(source, 'owner', None),
)
# Copy messages up to keep_count
msgs_to_copy = source.history[:keep_count]
for msg in msgs_to_copy:
# Copy the metadata dict. Sharing it would let the fork's
# persistence (add_message -> _persist_message stamps
# _db_id/timestamp onto the dict) mutate the SOURCE session's
# in-memory messages, corrupting their _db_id and breaking
# edit/delete-by-id on the original conversation.
meta = dict(msg.metadata) if isinstance(msg.metadata, dict) else None
new_session.add_message(ChatMessage(msg.role, msg.content, meta))
try:
from src.event_bus import fire_event
fire_event("session_created", getattr(source, 'owner', None))
except Exception:
logger.debug("session_created event dispatch failed", exc_info=True)
return {
"status": "ok",
"id": new_id,
"name": fork_name,
"kept": len(msgs_to_copy),
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Fork error {session_id}: {e}")
raise HTTPException(500, str(e))
@router.get("/api/conversations/topics")
async def get_conversation_topics(request: Request) -> Dict[str, Any]:
from src.auth_helpers import require_user
user = require_user(request)
try:
return analyze_topics(session_manager, owner=user or None)
except Exception as e:
raise HTTPException(500, f"Topic analysis failed: {e}")
@router.post("/api/session/{session_id}/compact")
async def compact_session(request: Request, session_id: str):
"""Manually trigger context compaction for a session."""
_verify_session_owner(request, session_id)
from src.auth_helpers import effective_user
owner = effective_user(request)
try:
session = session_manager.get_session(session_id)
except KeyError:
raise HTTPException(404, "Session not found")
_reject_compact_during_active_run(session_id)
try:
from src.model_context import estimate_tokens, get_context_length
from src.llm_core import llm_call_async
from src.endpoint_resolver import resolve_endpoint
if len(session.history) < 6:
return {"status": "ok", "message": "Not enough messages to compact"}
ctx_len = get_context_length(session.endpoint_url, session.model)
messages_before = session.get_context_messages()
used_before = estimate_tokens(messages_before)
pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
msg_count_before = len(session.history)
# Keep only last 4 messages, summarize the rest
keep_count = 4
older = session.history[:-keep_count]
recent = session.history[-keep_count:]
# Build text to summarize
convo_text = "\n".join(
f"{_message_role(m).upper()}: "
f"{_message_text(m)[:2000]}"
for m in older
)
# Use utility model if available
util_url, util_model, util_headers = resolve_endpoint("utility", owner=owner or None)
compact_url = util_url or session.endpoint_url
compact_model = util_model or session.model
compact_headers = util_headers if util_url else session.headers
from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
summary = await llm_call_async(
compact_url, compact_model,
[
{"role": "system", "content": sys_prompt},
{"role": "user", "content": convo_text},
],
temperature=0.2, max_tokens=1024,
headers=compact_headers, timeout=30,
)
# Replace session history: summary as system message + recent messages
# System message holds the full summary for AI context
system_summary = ChatMessage(
role="system",
content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
metadata={"compacted": True, "hidden": True},
)
# Visible assistant message just shows stats
summary_msg = ChatMessage(
role="assistant",
content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
metadata={"compacted": True, "messages_removed": len(older)},
)
new_history = [system_summary, summary_msg] + list(recent)
session.history = new_history
session.message_count = len(session.history)
logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
# Update DB: delete old messages, insert summary
db = SessionLocal()
try:
db_msgs = db.query(DbChatMessage).filter(
DbChatMessage.session_id == session_id
).order_by(DbChatMessage.timestamp).all()
# Delete all but the last keep_count
for m in db_msgs[:-keep_count]:
db.delete(m)
# Insert system summary (hidden, for AI context) and visible summary
import json as _json
import uuid
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
db_sys_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="system",
content=system_summary.content,
meta_data=_json.dumps(system_summary.metadata),
timestamp=now,
)
db.add(db_sys_summary)
db_summary = DbChatMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role="assistant",
content=summary_msg.content,
meta_data=_json.dumps(summary_msg.metadata),
timestamp=now,
)
db.add(db_summary)
# Update session record
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.message_count = len(session.history)
db_session.updated_at = datetime.now(timezone.utc)
db.commit()
finally:
db.close()
session_manager.save_sessions()
used_after = estimate_tokens(session.get_context_messages())
pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
return {
"status": "ok",
"message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
"before": pct_before,
"after": pct_after,
}
except Exception as e:
logger.error(f"Manual compact error {session_id}: {e}")
raise HTTPException(500, str(e))
return router
_sys.modules[__name__] = _canonical
routes/note_routes.py
+16 -5
View File
@@ -483,11 +483,22 @@ async def dispatch_reminder(
api_key = intg.get("api_key", "")
if api_key:
hdrs["Authorization"] = f"Bearer {api_key}"
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
ntfy_sent = resp.is_success
if not ntfy_sent:
ntfy_error = f"ntfy returned HTTP {resp.status_code}"
# SSRF guard — same check (and env knob) as the webhook branch
# above: link-local / metadata addresses are always rejected;
# REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
# so a ntfy base_url can't be pointed at internal services.
import os as _os
from src.url_safety import check_outbound_url as _chk
_block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
_ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
if not _ok:
ntfy_error = f"ntfy URL rejected: {_reason}"
else:
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
ntfy_sent = resp.is_success
if not ntfy_sent:
ntfy_error = f"ntfy returned HTTP {resp.status_code}"
else:
ntfy_error = "No enabled ntfy integration"
except Exception as e:
routes/research/research_routes.py
+85 -29
View File
@@ -20,6 +20,55 @@ from src.constants import DEEP_RESEARCH_DIR
_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
def _validate_session_id(session_id: str) -> str:
if not _SESSION_ID_RE.fullmatch(session_id):
raise HTTPException(400, "Invalid session ID format")
return session_id
def _research_storage_root() -> Path:
return Path(DEEP_RESEARCH_DIR).resolve()
def _find_research_path(session_id: str) -> Path | None:
"""Find a persisted research file without deriving its path from input."""
expected_name = f"{_validate_session_id(session_id)}.json"
root = _research_storage_root()
for stored_path in root.glob("*.json"):
if stored_path.name != expected_name:
continue
resolved = stored_path.resolve()
try:
resolved.relative_to(root)
except ValueError:
return None
if not resolved.is_file():
return None
return resolved
return None
def _require_research_path(session_id: str) -> Path:
path = _find_research_path(session_id)
if path is None:
raise HTTPException(404, "Research not found")
return path
def _find_owned_research_path(session_id: str, user: str) -> Path | None:
path = _find_research_path(session_id)
if path is None:
return None
try:
owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
except Exception:
return None
if owner != user:
return None
return path
logger = logging.getLogger(__name__)
# Model-name substrings that are NOT chat/generation models — research must
@@ -172,10 +221,6 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
raise HTTPException(401, "Not authenticated")
return user
def _validate_session_id(session_id: str) -> None:
if not _SESSION_ID_RE.fullmatch(session_id):
raise HTTPException(400, "Invalid session ID format")
def _owns_in_memory(session_id: str, user: str) -> bool:
"""Ownership check for an in-flight (in-memory) research task.
Falls back to the on-disk JSON if the task has already finished."""
@@ -183,14 +228,34 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
if entry is not None:
return entry.get("owner", "") == user
# Task no longer in memory — check the persisted JSON.
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
return False
try:
return json.loads(path.read_text(encoding="utf-8")).get("owner") == user
except Exception:
return _find_owned_research_path(session_id, user) is not None
except HTTPException:
return False
def _require_owned_or_active_research_path(session_id: str, user: str) -> Path | None:
"""Validate ownership once and return the completed on-disk path.
Active running research has no completed disk path yet. Completed
tasks can remain in _active_tasks after persistence, so prefer their
owned disk path when available. Completed disk lookups still reuse the
path after the ownership gate.
"""
entry = research_handler._active_tasks.get(session_id)
if entry is not None:
if entry.get("owner", "") != user:
raise HTTPException(404, "No research found for this session")
if entry.get("status") != "running":
path = _find_owned_research_path(session_id, user)
if path is not None:
return path
return None
path = _find_owned_research_path(session_id, user)
if path is None:
raise HTTPException(404, "No research found for this session")
return path
@router.get("/api/research/active")
async def research_active(request: Request):
"""List all currently active (running) research tasks."""
@@ -247,9 +312,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
def _assert_owns_research(session_id: str, user: str) -> None:
"""404-not-403 ownership gate for a research session's on-disk JSON.
Use BEFORE returning any data or mutating the file."""
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
raise HTTPException(404, "Research not found")
path = _require_research_path(session_id)
try:
owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
except Exception:
@@ -361,9 +424,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
summary, stats — used by the Library preview panel."""
user = _require_user(request)
_validate_session_id(session_id)
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
raise HTTPException(404, "Research not found")
path = _require_research_path(session_id)
try:
data = json.loads(path.read_text(encoding="utf-8"))
except Exception as e:
@@ -378,9 +439,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
"""Soft-archive / restore a research report (sets `archived` in its JSON)."""
user = _require_user(request)
_validate_session_id(session_id)
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if not path.exists():
raise HTTPException(404, "Research not found")
path = _require_research_path(session_id)
try:
data = json.loads(path.read_text(encoding="utf-8"))
if data.get("owner") != user:
@@ -398,10 +457,9 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
"""Delete a research result from disk."""
user = _require_user(request)
_validate_session_id(session_id)
data_dir = Path(DEEP_RESEARCH_DIR)
json_path = data_dir / f"{session_id}.json"
json_path = _find_research_path(session_id)
deleted = False
if json_path.exists():
if json_path is not None:
# SECURITY: verify ownership before letting the caller delete it.
try:
data = json.loads(json_path.read_text(encoding="utf-8"))
@@ -557,12 +615,11 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
"""Get research result without clearing it (for panel use)."""
user = _require_user(request)
_validate_session_id(session_id)
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
owned_disk_path = _require_owned_or_active_research_path(session_id, user)
result = research_handler.get_result(session_id)
if result is None:
p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if p.exists():
p = owned_disk_path
if p is not None:
d = json.loads(p.read_text(encoding="utf-8"))
return {
"result": d.get("result", ""),
@@ -591,8 +648,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
# otherwise any authenticated user could spin off (and thereby read)
# another user's report by guessing its session ID. Mirrors every other
# endpoint in this file (see result_peek above).
if not _owns_in_memory(session_id, user):
raise HTTPException(404, "No research found for this session")
owned_disk_path = _require_owned_or_active_research_path(session_id, user)
if session_manager is None:
raise HTTPException(500, "session_manager not configured")
@@ -601,8 +657,8 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
sources = research_handler.get_sources(session_id) or []
query = ""
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json"
if path.exists():
path = owned_disk_path
if path is not None:
try:
disk = json.loads(path.read_text(encoding="utf-8"))
if not result:
routes/session_routes.py
+10 -10
View File
@@ -162,7 +162,7 @@ def _persist_session_headers(session_id: str, headers: dict | None) -> None:
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.headers = headers or {}
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
db.commit()
except Exception:
db.rollback()
@@ -224,8 +224,8 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
# purge exists only to catch ghosts the frontend missed (tab close,
# crash). Only clean up rows old enough to be definitely orphaned.
try:
from datetime import datetime as _dt, timedelta as _td
_cutoff = _dt.utcnow() - _td(minutes=10)
from datetime import timedelta as _td
_cutoff = utcnow_naive() - _td(minutes=10)
_purge_db = SessionLocal()
try:
from core.database import ChatMessage as _DbMsg
@@ -471,7 +471,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
db_session = db.query(DbSession).filter(DbSession.id == sid).first()
if db_session:
db_session.folder = folder if folder else None
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
db.commit()
result["folder"] = folder if folder else None
finally:
@@ -518,7 +518,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
db_session.model = model
db_session.endpoint_url = endpoint_url
db_session.headers = session.headers or {}
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
db.commit()
finally:
db.close()
@@ -647,7 +647,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
db_session = db.query(DbSession).filter(DbSession.id == sid).first()
if db_session:
db_session.archived = True
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
db.commit()
# Update in memory if it exists
@@ -681,7 +681,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
if not db_session:
raise HTTPException(404, f"Session {sid} not found")
db_session.archived = False
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
db.commit()
# Reload into session manager so it appears in the active list
try:
@@ -891,7 +891,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
if db_session:
db_session.is_important = important
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
db.commit()
# Update in memory if it exists
@@ -980,7 +980,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
metadata={
"compacted": True,
"summarized_count": len(older),
"timestamp": datetime.utcnow().isoformat(),
"timestamp": utcnow_naive().isoformat(),
},
)
new_history = [summary_msg] + recent
@@ -1257,7 +1257,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
db_session = db_session_q.first()
if db_session:
db_session.folder = folder_name
db_session.updated_at = datetime.utcnow()
db_session.updated_at = utcnow_naive()
updated += 1
db.commit()
except Exception as e:
routes/task_routes.py
+28 -24
View File
@@ -11,10 +11,14 @@ from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel
from core.database import SessionLocal, ScheduledTask, TaskRun
from core.middleware import INTERNAL_TOOL_USER
from core.constants import internal_api_base
from src.auth_helpers import get_current_user
from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
from src.task_action_policy import (
ADMIN_ONLY_TASK_ACTIONS,
is_admin_only_task_action,
owner_has_admin_task_privileges,
)
from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS
from routes.prefs_routes import _load_for_user, _save_for_user
@@ -417,28 +421,18 @@ def setup_task_routes(task_scheduler) -> APIRouter:
db.close()
return {"ok": True, "opened": True, "enabled": bool(prefs.get("tasks_enabled")), "resumed": resumed}
# Actions that execute shell/SSH commands — restricted to admins.
# Actions that execute shell/SSH commands or cross into admin-only
# Cookbook serving surfaces — restricted to admins.
# Non-admin users cannot create tasks with these action types via the
# API. See review CRIT-C.
_ADMIN_ONLY_ACTIONS = {"run_local", "run_script", "ssh_command"}
_ADMIN_ONLY_ACTIONS = ADMIN_ONLY_TASK_ACTIONS
def _is_admin(user: str | None) -> bool:
if not user:
return False
# In-process tool-loopback marker — AuthMiddleware validated
# the internal token + loopback client before stamping this,
# so treat as admin-equivalent.
if user == INTERNAL_TOOL_USER:
return True
try:
from core.auth import AuthManager
auth = AuthManager()
if not auth.is_configured:
# Unconfigured single-user deploy: trust the local owner.
return True
return bool(auth.is_admin(user))
except Exception:
return False
return owner_has_admin_task_privileges(user)
def _require_admin_for_task_action(user: str | None, task_type: str | None, action: str | None) -> None:
if is_admin_only_task_action(task_type, action) and not _is_admin(user):
raise HTTPException(403, f"Action '{action}' requires admin privileges")
def _validate_then_task_id(db, then_task_id: Optional[str], user: Optional[str], current_task_id: Optional[str] = None) -> Optional[str]:
target_id = (then_task_id or "").strip()
@@ -466,8 +460,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
# Block shell-executing action types for non-admins. action_run_local
# uses subprocess.run(shell=True) and ssh_command / run_script run
# arbitrary commands.
if req.task_type == "action" and req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
_require_admin_for_task_action(user, req.task_type, req.action)
if req.trigger_type == "schedule" and not req.schedule:
raise HTTPException(400, "Schedule is required for schedule-triggered tasks")
if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression:
@@ -681,6 +674,10 @@ def setup_task_routes(task_scheduler) -> APIRouter:
if user and task.owner != user:
raise HTTPException(403, "Access denied")
next_task_type = req.task_type if req.task_type is not None else task.task_type
next_action = req.action if req.action is not None else task.action
_require_admin_for_task_action(user, next_task_type, next_action)
if req.name is not None:
task.name = req.name
if req.prompt is not None:
@@ -688,9 +685,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
if req.task_type is not None:
task.task_type = req.task_type
if req.action is not None:
# Same admin-only gate as create — see CRIT-C.
if req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
task.action = req.action
if req.output_target is not None:
task.output_target = req.output_target
@@ -807,6 +801,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
raise HTTPException(404, "Task not found")
if user and task.owner != user:
raise HTTPException(403, "Access denied")
_require_admin_for_task_action(user, task.task_type, task.action)
task.status = "active"
if (task.trigger_type or "schedule") == "schedule":
task.next_run = compute_next_run(
@@ -869,6 +864,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
raise HTTPException(404, "Task not found")
if user and task.owner != user:
raise HTTPException(403, "Access denied")
_require_admin_for_task_action(user, task.task_type, task.action)
finally:
db.close()
started = await task_scheduler.run_task_now(task_id, force=force)
@@ -1058,6 +1054,14 @@ def setup_task_routes(task_scheduler) -> APIRouter:
).first()
if not task:
raise HTTPException(404, "Not found")
if (
is_admin_only_task_action(task.task_type, task.action)
and not owner_has_admin_task_privileges(task.owner)
):
task.status = "paused"
task.next_run = None
db.commit()
raise HTTPException(403, f"Action '{task.action}' requires admin privileges")
finally:
db.close()
started = await task_scheduler.run_task_now(task_id)