diff --git a/app.py b/app.py index 57d091efd..d7045b100 100644 --- a/app.py +++ b/app.py @@ -1001,17 +1001,21 @@ async def _startup_event(): _startup_tasks.append(asyncio.create_task(_warmup_endpoints())) - # Keep-alive: ping endpoints every 60 seconds to prevent cold starts - async def _keepalive_loop(): - while True: - try: - await asyncio.sleep(60) - await _warmup_endpoints() - except Exception as e: - logger.warning(f"Keepalive loop error: {e}") - await asyncio.sleep(300) # Back off on error + # Keep-alive is opt-in. The ping path performs model discovery, and when + # stale LAN endpoints are configured it can add periodic backend pressure + # that delays unrelated UI requests such as Notes/Documents. + _keepalive_enabled = str(os.getenv("ODYSSEUS_MODEL_KEEPALIVE", "")).lower() in {"1", "true", "yes", "on"} + if _keepalive_enabled: + async def _keepalive_loop(): + while True: + try: + await asyncio.sleep(60) + await _warmup_endpoints() + except Exception as e: + logger.warning(f"Keepalive loop error: {e}") + await asyncio.sleep(300) # Back off on error - _startup_tasks.append(asyncio.create_task(_keepalive_loop())) + _startup_tasks.append(asyncio.create_task(_keepalive_loop())) async def _ensure_default_tasks(): # Create/reconcile default automation tasks + personal assistant for every user. diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index 2611491ae..96c985864 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -538,6 +538,148 @@ def _get_cached_summaries(): return {} +def _fixture_email_file() -> Path: + return DATA_DIR / "fixture_email_messages.json" + + +def _fixture_email_enabled() -> bool: + return _fixture_email_file().exists() + + +def _parse_fixture_date(raw_date: str) -> tuple[str, float]: + if not raw_date: + return "", 0.0 + parsed = None + try: + parsed = datetime.fromisoformat(str(raw_date).replace("Z", "+00:00")) + except Exception: + try: + parsed = email.utils.parsedate_to_datetime(str(raw_date)) + except Exception: + parsed = None + if parsed: + return parsed.isoformat(), parsed.timestamp() + return str(raw_date), 0.0 + + +def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict: + sender = str(row.get("from") or "Fixture Sender ") + sender_name, sender_addr = email.utils.parseaddr(sender) + date_str, date_epoch = _parse_fixture_date(str(row.get("date") or "")) + subject = str(row.get("subject") or "(no subject)") + body = str(row.get("body") or "") + owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default") + uid = str(uid_num) + return { + "uid": uid, + "message_id": f"", + "subject": subject, + "from": sender_name or sender_addr or sender, + "from_address": sender_addr, + "date": date_str, + "date_epoch": date_epoch, + "summary": body[:240], + "body": body, + "account": "Fixture Inbox", + "account_email": owner or str(row.get("owner") or ""), + "account_id": "fixture-email", + "attachments": [], + } + + +def _fixture_email_rows(owner: str | None = None) -> list[dict]: + path = _fixture_email_file() + if not path.exists(): + return [] + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return [] + rows = raw.get("messages") if isinstance(raw, dict) else raw + out = [] + owner = str(owner or "").strip() + for i, row in enumerate(rows if isinstance(rows, list) else [], start=1): + if not isinstance(row, dict): + continue + row_owner = str(row.get("owner") or "").strip() + if owner and row_owner and row_owner != owner: + continue + out.append(_fixture_email_record(row, i, owner or row_owner)) + out.sort(key=lambda item: item.get("date_epoch") or 0, reverse=True) + return out + + +def _fixture_account_rows() -> list[dict]: + if not _fixture_email_enabled(): + return [] + owner = _current_owner() + owners = [] + for row in _fixture_email_rows(owner or None): + email_addr = row.get("account_email") or owner or "fixture@fixtures.odysseus.local" + if email_addr not in owners: + owners.append(email_addr) + if not owners: + owners = [owner or "fixture@fixtures.odysseus.local"] + return [ + { + "id": "fixture-email", + "owner": owner or owners[0], + "name": "Fixture Inbox", + "is_default": True, + "imap_user": owners[0], + "from_address": owners[0], + } + ] + + +def _fixture_email_matches(item: dict, query: str) -> bool: + if not query: + return True + terms = [term for term in re.split(r"\W+", str(query).lower()) if term] + haystack = "\n".join( + str(item.get(key) or "") + for key in ("subject", "from", "from_address", "body", "summary") + ).lower() + return all(term in haystack for term in terms) + + +def _fixture_list_emails(folder="INBOX", max_results=20, unresponded_only=False, + unread_only=False, account=None) -> list[dict] | None: + if not _fixture_email_enabled(): + return None + if account and str(account).strip().lower() not in { + "fixture-email", + "fixture inbox", + "fixture", + str(_current_owner()).lower(), + }: + return [] + if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}: + return [] + return _fixture_email_rows(_current_owner())[: int(max_results or 20)] + + +def _fixture_search_emails(query, folders=None, max_results=20, account=None) -> list[dict] | None: + if not _fixture_email_enabled(): + return None + rows = _fixture_list_emails("INBOX", max_results=1000, account=account) or [] + out = [dict(row, _folder="INBOX") for row in rows if _fixture_email_matches(row, str(query or ""))] + return out[: int(max_results or 20)] + + +def _fixture_read_email(uid=None, message_id=None, folder="INBOX", account=None) -> dict | None: + if not _fixture_email_enabled(): + return None + if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}: + return {"error": f"Email UID {uid or message_id} not found"} + for item in _fixture_email_rows(_current_owner()): + if uid and str(item.get("uid")) == str(uid): + return item + if message_id and str(item.get("message_id")) == str(message_id): + return item + return {"error": f"Email not found with UID/Message-ID: {uid or message_id}"} + + # ── Tool implementations ── @@ -548,6 +690,9 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False, Pass unread_only=True and/or unresponded_only=True for attention scans. account selects mailbox (None = default). """ + fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, account) + if fixture is not None: + return fixture conn = None try: conn = _imap_connect(account) @@ -629,6 +774,9 @@ def _result_sort_time(result: dict) -> datetime: def _list_emails_across_accounts(folder="INBOX", max_results=20, unresponded_only=False, unread_only=False): + fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, None) + if fixture is not None: + return fixture, [] rows = _list_accounts_raw() combined = [] errors = [] @@ -662,6 +810,9 @@ def _search_emails(query, folders=None, max_results=20, account=None): _list_emails plus an `_folder` tag.""" if not query or not str(query).strip(): return [] + fixture = _fixture_search_emails(query, folders=folders, max_results=max_results, account=account) + if fixture is not None: + return fixture q = str(query).replace("\\", "\\\\").replace('"', '\\"') # Mail clients commonly use OR FROM/SUBJECT/TEXT to match either field. # IMAP SEARCH OR is binary, so we nest it. @@ -784,6 +935,9 @@ def _extract_attachment_to_disk(msg, index, target_dir): def _read_email(uid=None, message_id=None, folder="INBOX", account=None): """Read full email content by UID or message-ID. account = mailbox selector.""" + fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=account) + if fixture is not None: + return fixture cfg = _load_config(account) conn = None try: @@ -837,6 +991,9 @@ def _read_email(uid=None, message_id=None, folder="INBOX", account=None): def _read_email_across_accounts(uid=None, message_id=None, folder="INBOX"): + fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=None) + if fixture is not None: + return fixture rows = _list_accounts_raw() matches = [] errors = [] @@ -1775,9 +1932,10 @@ async def list_tools() -> list[Tool]: Tool( name="reply_to_email", description=( - "Reply to an existing email by UID. This sends immediately; for normal " - "assistant-written replies, prefer draft_email_reply so the user can " - "review and send from Odysseus. Automatically threads the reply with " + "Reply to an existing email by UID. This sends immediately. Do NOT use " + "for normal 'write/draft a reply saying X' requests; use " + "draft_email_reply so the user can review and send from Odysseus. " + "Only use this when the user explicitly says to send now. Automatically threads the reply with " "In-Reply-To and References headers, prefixes 'Re:' on the subject, and " "uses the original sender as the recipient. Set reply_all=true to also CC " "the original To/Cc recipients. For follow-up 'reply ...' requests, use " @@ -1991,6 +2149,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: if name == "list_email_accounts": rows = _filter_accounts_for_owner(all_db_accounts) + if not rows: + rows = _fixture_account_rows() if not rows: if all_db_accounts and owner: return [TextContent(type="text", text="No email accounts configured for this owner.")] diff --git a/routes/chat_routes.py b/routes/chat_routes.py index b4a6ed837..225c83e28 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -789,19 +789,19 @@ def setup_chat_routes( "manage_skills", # skill presets tied to user }) - # Active email reader open → strip the tools that let the agent - # "drift" to a new compose: create_document (writes a fake email- - # shaped .md file) and send_email (sends fresh to a recipient the - # agent invented). With those gone, the only paths left for "write - # email saying X" are ui_control open_email_reply (draft) and - # reply_to_email (immediate send) — both of which use the open - # email's UID. Code-level enforcement instead of relying on a - # prompt rule the model can ignore. + # Active email reader open → strip the tools that let the agent drift + # away from the visible email or skip review. The only allowed compose + # path is ui_control open_email_reply, which opens the same draft editor + # as the Reply button with the generated body pre-filled. This prevents + # the model from falling back to direct SMTP when it botches a draft + # call, and prevents fake email-shaped documents. if active_email_ctx and active_email_ctx.get("uid"): disabled_tools.update({ "create_document", "send_email", + "reply_to_email", "mcp__email__send_email", + "mcp__email__reply_to_email", }) # Enforce per-user privileges diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index f57ecf6e5..ea4eb2f19 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -9,6 +9,7 @@ import shlex import shutil import subprocess import sys +import urllib.request import uuid from pathlib import Path @@ -400,6 +401,35 @@ def setup_cookbook_routes() -> APIRouter: safe_chmod(key_path.with_suffix(".pub"), 0o644) return {"ok": True, "public_key": _read_cookbook_public_key()} + class CookbookSshTestRequest(BaseModel): + host: str + ssh_port: str | None = None + + @router.post("/api/cookbook/test-ssh") + async def test_cookbook_ssh(request: Request, req: CookbookSshTestRequest): + """Test a configured Cookbook SSH target without using generic shell exec.""" + require_admin(request) + host = validate_remote_host(req.host) + ssh_port = validate_ssh_port(req.ssh_port) + try: + code, stdout, stderr = await run_ssh_command_async( + host, + ssh_port, + "echo ok", + timeout=8, + connect_timeout=5, + strict_host_key_checking=False, + ) + except asyncio.TimeoutError: + return {"stdout": "", "stderr": "SSH test timed out", "exit_code": 124} + except Exception as e: + return {"stdout": "", "stderr": str(e), "exit_code": -1} + return { + "stdout": stdout.decode("utf-8", errors="replace"), + "stderr": stderr.decode("utf-8", errors="replace"), + "exit_code": code, + } + def _needs_binary(cmd: str, binary: str) -> bool: return bool(re.search(rf"(^|[\s;&|()]){re.escape(binary)}($|[\s;&|()])", cmd or "")) @@ -1119,6 +1149,22 @@ def setup_cookbook_routes() -> APIRouter: try: ep = db.query(_ME).filter(_ME.id == endpoint_id).first() if ep: + # A scheduled serve can leave old non-zero exit markers + # in tmux scrollback while the current OpenAI endpoint is + # actually alive. Verify reachability before deleting the + # endpoint row; otherwise chats fall back even though the + # served model is ready. + try: + probe_url = ep.base_url.rstrip("/") + "/models" + with urllib.request.urlopen(probe_url, timeout=3) as resp: + if 200 <= getattr(resp, "status", 0) < 300: + logger.info( + f"crash-watchdog: serve {session_id} has exit marker {exit_code} " + f"but endpoint {ep.id} is reachable; leaving it registered" + ) + return + except Exception: + pass logger.info( f"crash-watchdog: dropping endpoint {endpoint_id} " f"({ep.name} @ {ep.base_url}) — serve exited {exit_code}" @@ -1205,6 +1251,8 @@ def setup_cookbook_routes() -> APIRouter: existing.is_enabled = True existing.model_type = "llm" existing.name = display_name + existing.endpoint_kind = "local" + existing.model_refresh_mode = "auto" if is_ollama_endpoint: existing.endpoint_kind = "ollama" if pinned_models: @@ -1252,7 +1300,8 @@ def setup_cookbook_routes() -> APIRouter: api_key=None, is_enabled=True, model_type="llm", - endpoint_kind="ollama" if is_ollama_endpoint else "auto", + endpoint_kind="ollama" if is_ollama_endpoint else "local", + model_refresh_mode="auto", cached_models=json.dumps(pinned_models) if pinned_models else None, pinned_models=json.dumps(pinned_models) if pinned_models else None, supports_tools=supports_tools, diff --git a/routes/document_helpers.py b/routes/document_helpers.py index 0de4cc2a3..21a872ba6 100644 --- a/routes/document_helpers.py +++ b/routes/document_helpers.py @@ -28,6 +28,7 @@ class DocumentCreate(BaseModel): class DocumentUpdate(BaseModel): content: str summary: Optional[str] = None + force_version: bool = False class DocumentPatch(BaseModel): title: Optional[str] = None diff --git a/routes/document_routes.py b/routes/document_routes.py index d35d2a79e..dd173f64a 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -569,8 +569,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: raise HTTPException(404, "Document not found") _verify_doc_owner(db, doc, user) - # Skip if content is identical - if doc.current_content == req.content: + # Skip if content is identical unless the caller explicitly wants + # a checkpoint version from the current editor state. + if doc.current_content == req.content and not req.force_version: return _doc_to_dict(doc) _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler) @@ -582,7 +583,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: now = datetime.now(timezone.utc) coalesced = False - if latest_ver and latest_ver.source == "user": + if latest_ver and latest_ver.source == "user" and not req.force_version: ver_time = latest_ver.created_at if ver_time.tzinfo is None: ver_time = ver_time.replace(tzinfo=timezone.utc) @@ -798,10 +799,21 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: from src.document_actions import _JUNK_TITLES to_delete = [] + now = datetime.now(timezone.utc) for doc in docs: content = (doc.current_content or "").strip() title_raw = (doc.title or "").strip() title = title_raw.lower() + created = doc.created_at + if created and created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + is_fresh_empty = ( + not content + and created is not None + and (now - created).total_seconds() < 1800 + ) + if is_fresh_empty: + continue # Strip markdown noise to get a "real" character count stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE) diff --git a/routes/email_helpers.py b/routes/email_helpers.py index 513ec1f0a..9bfa71b02 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -433,6 +433,19 @@ def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, co info = conn.execute(f"PRAGMA table_info({table})").fetchall() cols = [r[1] for r in info] pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])] + for col in columns: + if col not in cols: + if col == "owner": + conn.execute(f"ALTER TABLE {table} ADD COLUMN owner TEXT DEFAULT ''") + elif col in {"event_uids"}: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT '[]'") + elif col.startswith("has_") or col.endswith("_created") or col.endswith("_count"): + conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} INTEGER DEFAULT 0") + elif col == "created_at": + conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT ''") + else: + conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT") + cols.append(col) if "owner" in cols and pk_cols == ["message_id", "owner"]: return @@ -575,6 +588,7 @@ def _init_scheduled_db(): CREATE TABLE IF NOT EXISTS email_tags ( message_id TEXT, owner TEXT DEFAULT '', + account_id TEXT DEFAULT '', uid TEXT, folder TEXT, subject TEXT, @@ -585,7 +599,7 @@ def _init_scheduled_db(): moved_to TEXT, model_used TEXT, created_at TEXT NOT NULL, - PRIMARY KEY (message_id, owner) + PRIMARY KEY (message_id, owner, account_id) ) """) # Backfill migration: older installs created the table with @@ -593,28 +607,35 @@ def _init_scheduled_db(): # promote it into the PK by rebuild-copy-swap (SQLite can't ALTER PK). try: _cols = [r[1] for r in conn.execute("PRAGMA table_info(email_tags)")] + _pk_cols = [r[1] for r in sorted(conn.execute("PRAGMA table_info(email_tags)").fetchall(), key=lambda row: row[5] or 99) if r[5]] if "owner" not in _cols: - # Add the column first so reads/writes don't break mid-migration. conn.execute("ALTER TABLE email_tags ADD COLUMN owner TEXT DEFAULT ''") - # Rebuild with composite PK. Existing rows get owner='' (legacy - # single-user); the urgency scanner will overwrite as it - # re-classifies. No data loss. + _cols.append("owner") + if "account_id" not in _cols: + conn.execute("ALTER TABLE email_tags ADD COLUMN account_id TEXT DEFAULT ''") + _cols.append("account_id") + if _pk_cols != ["message_id", "owner", "account_id"]: + # Rebuild with account-aware composite PK. Existing rows get + # account_id='' and are still readable as legacy fallback rows; + # fresh task runs write exact account ids and no longer block each + # other when two accounts share a Message-ID. conn.execute(""" CREATE TABLE IF NOT EXISTS email_tags__new ( message_id TEXT, owner TEXT DEFAULT '', + account_id TEXT DEFAULT '', uid TEXT, folder TEXT, subject TEXT, sender TEXT, tags TEXT, spam_verdict INTEGER DEFAULT 0, spam_reason TEXT, moved_to TEXT, model_used TEXT, created_at TEXT NOT NULL, - PRIMARY KEY (message_id, owner) + PRIMARY KEY (message_id, owner, account_id) ) """) conn.execute(""" INSERT OR IGNORE INTO email_tags__new - (message_id, owner, uid, folder, subject, sender, tags, + (message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict, spam_reason, moved_to, model_used, created_at) - SELECT message_id, COALESCE(owner, ''), uid, folder, subject, + SELECT message_id, COALESCE(owner, ''), COALESCE(account_id, ''), uid, folder, subject, sender, tags, spam_verdict, spam_reason, moved_to, model_used, created_at FROM email_tags @@ -630,11 +651,12 @@ def _init_scheduled_db(): message_id TEXT, owner TEXT DEFAULT '', uid TEXT, + event_uids TEXT DEFAULT '[]', events_created INTEGER DEFAULT 0, created_at TEXT NOT NULL, PRIMARY KEY (message_id, owner) ) - """, ["message_id", "owner", "uid", "events_created", "created_at"]) + """, ["message_id", "owner", "uid", "event_uids", "events_created", "created_at"]) _ensure_owner_scoped_email_cache_table(conn, "email_urgency_alerts", """ CREATE TABLE IF NOT EXISTS email_urgency_alerts ( message_id TEXT, @@ -660,6 +682,64 @@ def _init_scheduled_db(): PRIMARY KEY (owner, account_key, folder, message_key) ) """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS email_message_index ( + owner TEXT NOT NULL DEFAULT '', + account_key TEXT NOT NULL DEFAULT '', + folder TEXT NOT NULL, + uid TEXT NOT NULL, + message_id TEXT, + subject TEXT, + from_name TEXT, + from_address TEXT, + to_text TEXT, + cc_text TEXT, + date_iso TEXT, + date_display TEXT, + date_epoch REAL DEFAULT 0, + size INTEGER DEFAULT 0, + flags TEXT DEFAULT '', + has_attachments INTEGER DEFAULT 0, + updated_at TEXT NOT NULL, + PRIMARY KEY (owner, account_key, folder, uid) + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date + ON email_message_index(owner, account_key, folder, date_epoch DESC) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS ix_email_message_index_message_id + ON email_message_index(owner, account_key, message_id) + """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS email_body_preview_cache ( + owner TEXT NOT NULL DEFAULT '', + account_key TEXT NOT NULL DEFAULT '', + folder TEXT NOT NULL, + uid TEXT NOT NULL, + message_id TEXT, + payload_json TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (owner, account_key, folder, uid) + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS ix_email_body_preview_message_id + ON email_body_preview_cache(owner, account_key, message_id) + """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS email_attachment_metadata_cache ( + owner TEXT NOT NULL DEFAULT '', + account_key TEXT NOT NULL DEFAULT '', + folder TEXT NOT NULL, + uid TEXT NOT NULL, + message_id TEXT, + attachments_json TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (owner, account_key, folder, uid) + ) + """) # Boundary cache — LLM-detected sig/quote start positions in the body. # Stored as char offsets (-1 = no boundary found). Once cached, the # client uses these to fold without ever re-calling the LLM. @@ -1257,12 +1337,14 @@ def _list_attachments_from_msg(msg): except Exception: payload = b"" size = len(payload) if payload is not None else 0 + content_id = (part.get("Content-ID") or "").strip().strip("<>") attachments.append({ "index": idx, "filename": filename, "content_type": ct, "size": size, "is_inline": "inline" in cd.lower(), + "content_id": content_id, }) idx += 1 return attachments @@ -1701,6 +1783,10 @@ class SendEmailRequest(BaseModel): attachments: Optional[List[str]] = None # Which account to send from. None = default account. account_id: Optional[str] = None + # Source message for replies. When present, /send marks this exact message + # answered after successful delivery so it leaves undone/reply-soon views. + source_uid: Optional[str] = None + source_folder: Optional[str] = None # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled). odysseus_kind: Optional[str] = None # If true, /send waits for SMTP + Sent append and returns the sent UID. diff --git a/routes/email_pollers.py b/routes/email_pollers.py index 146db0ed7..f09dd1eeb 100644 --- a/routes/email_pollers.py +++ b/routes/email_pollers.py @@ -29,7 +29,7 @@ from datetime import datetime from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart -from src.llm_core import llm_call_async +from src.task_endpoint import resolve_task_candidates, task_llm_call_async from routes.email_helpers import ( _strip_think, _extract_reply, _apply_email_style_mechanics, _load_settings, _save_settings, _get_email_config, @@ -45,6 +45,35 @@ from routes.email_helpers import ( logger = logging.getLogger(__name__) +def _extract_json_array_from_text(text: str): + """Return the last valid JSON array embedded in model output, if any.""" + if not text: + return None + cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip(), flags=re.MULTILINE).strip() + decoder = json.JSONDecoder() + try: + parsed = decoder.decode(cleaned) + if isinstance(parsed, list): + return parsed + except Exception: + pass + + # Models often explain themselves and finish with `[]` or `[{"action":...}]`. + # Scan every array opener and keep the last complete JSON array, rather than + # using a greedy regex that can swallow prose containing square brackets. + last = None + for idx, ch in enumerate(cleaned): + if ch != "[": + continue + try: + parsed, _end = decoder.raw_decode(cleaned[idx:]) + except Exception: + continue + if isinstance(parsed, list): + last = parsed + return last + + def _owner_for_email_account(account_id: str | None) -> str: if not account_id: return "" @@ -77,6 +106,8 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru do_tag: bool = False, do_spam: bool = False, do_calendar: bool = False, days_back: int = 1, + account_id: str | None = None, + max_process: int | None = None, progress_cb=None) -> str: """One iteration of the email scan. Temporarily flips settings flags so the existing background-loop logic runs exactly once for the requested ops.""" @@ -91,7 +122,12 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru settings["email_auto_calendar"] = bool(do_calendar) _save_settings(settings) try: - return await _auto_summarize_pass(days_back=days_back, progress_cb=progress_cb) + return await _auto_summarize_pass( + days_back=days_back, + account_id=account_id, + max_process=max_process, + progress_cb=progress_cb, + ) finally: s2 = _load_settings() for k, v in prev.items(): @@ -129,7 +165,7 @@ def _latest_inbox_fallback_uids(conn, reconnect): return [], reconnect() -async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str: +async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str: """Single pass of the auto-summarize/reply scan. When account_id is None, iterates over every enabled account in @@ -156,28 +192,41 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None names = {} if len(ids) <= 1: # Single-account (or zero rows — fallback to legacy settings.json lookup) - return await _auto_summarize_pass_single(days_back=days_back, account_id=(ids[0] if ids else None), progress_cb=progress_cb) + return await _auto_summarize_pass_single( + days_back=days_back, + account_id=(ids[0] if ids else None), + max_process=max_process, + progress_cb=progress_cb, + ) outs = [] for idx, aid in enumerate(ids, start=1): try: await _emit_progress(progress_cb, f"{names.get(aid, aid[:8])}: starting ({idx}/{len(ids)})") - result = await _auto_summarize_pass_single(days_back=days_back, account_id=aid, progress_cb=progress_cb) + result = await _auto_summarize_pass_single( + days_back=days_back, + account_id=aid, + max_process=max_process, + progress_cb=progress_cb, + ) outs.append(f"[{names.get(aid, aid[:8])}] {result}") except Exception as e: logger.warning(f"auto-summarize pass failed for account {aid}: {e}") outs.append(f"[{names.get(aid, aid[:8])}] error: {e}") return "\n".join(outs) - return await _auto_summarize_pass_single(days_back=days_back, account_id=account_id, progress_cb=progress_cb) + return await _auto_summarize_pass_single( + days_back=days_back, + account_id=account_id, + max_process=max_process, + progress_cb=progress_cb, + ) -async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str: +async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str: """Single pass of the auto-summarize/reply scan for ONE account. Reads current settings flags.""" import asyncio import sqlite3 as _sql3 - import requests as _req - from src.endpoint_resolver import resolve_endpoint - from src.llm_core import _uses_max_completion_tokens, _restricts_temperature + from src.llm_core import _uses_max_completion_tokens settings = _load_settings() auto_sum = settings.get("email_auto_summarize", False) @@ -254,9 +303,15 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None ).fetchall()} if auto_tag or auto_spam: if account_owner: - _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner=?", (account_owner,)).fetchall()} + _tag_existing = {r[0] for r in _c.execute( + "SELECT message_id FROM email_tags WHERE owner=? AND (account_id=? OR account_id='' OR account_id IS NULL)", + (account_owner, account_id or ""), + ).fetchall()} else: - _tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner='' OR owner IS NULL").fetchall()} + _tag_existing = {r[0] for r in _c.execute( + "SELECT message_id FROM email_tags WHERE (owner='' OR owner IS NULL) AND (account_id=? OR account_id='' OR account_id IS NULL)", + (account_id or "",), + ).fetchall()} else: _tag_existing = set() _cal_existing = {r[0] for r in _c.execute( @@ -285,11 +340,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None if auto_spam and not spam_folder: logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move") - url, model, headers = resolve_endpoint("utility", owner=account_owner) - if not url: - url, model, headers = resolve_endpoint("default", owner=account_owner) - if not url or not model: + task_candidates = resolve_task_candidates(owner=account_owner) + if not task_candidates: return "No model configured" + url, model, headers = task_candidates[0] writing_style = settings.get("email_writing_style", "") processed = 0 @@ -303,7 +357,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None _reply_failed = 0 _detail_lines = [] _current_folder = "INBOX" - _max_process = 5 + # Calendar extraction is sequential and each row can involve a model + # call plus a calendar write. Keep the scheduled calendar-only pass + # below the 5-minute action budget instead of timing out mid-run. + _default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5 + try: + _max_process = max(1, int(max_process)) if max_process is not None else _default_max_process + except Exception: + _max_process = _default_max_process for _entry in uid_list: if processed >= _max_process: break @@ -395,48 +456,30 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None req_headers.update(headers) if need_sum: - tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" - payload = { - "model": model, - "messages": [ - {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<>>\n- ...\n<<>>\nAny reasoning or planning must come BEFORE <<>> (ideally inside ...). Only the text between the markers is kept."}, - {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<>> and <<>>."}, - ], - tok_key: 16384, - "temperature": 0.3, - "stream": False, - } - # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature. - if _restricts_temperature(model): - payload.pop("temperature", None) try: - # Use to_thread so this sync HTTP call doesn't freeze - # the entire event loop while the LLM thinks (240s). - resp = await asyncio.to_thread( - _req.post, url, json=payload, headers=req_headers, timeout=240 + summary = await task_llm_call_async( + messages=[ + {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<>>\n- ...\n<<>>\nAny reasoning or planning must come BEFORE <<>> (ideally inside ...). Only the text between the markers is kept."}, + {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<>> and <<>>."}, + ], + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=account_owner or None, + temperature=0.3, max_tokens=16384, timeout=240, ) - if resp.ok: - rdata = resp.json() - m = (rdata.get("choices") or [{}])[0].get("message", {}) - summary = (m.get("content") or "").strip() - summary = _extract_reply(summary) - if not summary: - rc = (m.get("reasoning_content") or "").strip() - bullets = [ln.strip() for ln in rc.split("\n") if re.match(r"^[-•*]\s+|^\d+[.)]\s+", ln.strip())] - summary = "\n".join(bullets) if bullets else "" - if summary: - _c = _sql3.connect(SCHEDULED_DB) - _c.execute(""" - INSERT OR REPLACE INTO email_summaries - (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat())) - _c.commit() - _c.close() - _sum_existing.add(message_id) - _summaries_created += 1 - _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) - _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") + summary = _extract_reply((summary or "").strip()) + if summary: + _c = _sql3.connect(SCHEDULED_DB) + _c.execute(""" + INSERT OR REPLACE INTO email_summaries + (message_id, owner, uid, folder, subject, sender, summary, model_used, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat())) + _c.commit() + _c.close() + _sum_existing.add(message_id) + _summaries_created += 1 + _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) + _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") except Exception as e: _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") @@ -457,14 +500,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None if context_snippets: sys_prompt += "\n\nRELEVANT CONTEXT FROM PAST EMAILS AND CONTACTS:\n" + "\n\n---\n\n".join(context_snippets[:5]) try: - reply = await llm_call_async( - url=url, model=model, + reply = await task_llm_call_async( messages=[ {"role": "system", "content": sys_prompt}, {"role": "user", "content": f"Original email:\nFrom: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\nDraft a reply. Return only the reply body text."}, ], - temperature=0.7, max_tokens=1024, - headers=req_headers, timeout=90, + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=account_owner or None, + temperature=0.7, max_tokens=1024, timeout=90, ) reply = _apply_email_style_mechanics(_extract_reply(reply or "")) if reply: @@ -491,6 +534,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None # ── Calendar event extraction (independent of reply drafting) ── if need_cal: _cal_run_count = 0 + _cal_event_uids = [] + _cal_parse_ok = False try: # Pull a snapshot of upcoming events so the LLM can decide # create vs update vs cancel based on what already exists. @@ -499,8 +544,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None _existing_summary = get_upcoming_events(_acct_owner, horizon_days=60, limit=40) existing_json = json.dumps(_existing_summary) is_sent = _folder.lower().startswith("sent") or "sent" in _folder.lower() - cal_extract = await llm_call_async( - url=url, model=model, + cal_extract = await task_llm_call_async( messages=[ {"role": "system", "content": ( "You are a calendar assistant. The user receives emails AND sends replies " @@ -551,8 +595,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None f"{body[:4000]}" )}, ], - temperature=0.1, max_tokens=16384, - headers=req_headers, timeout=180, + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=account_owner or None, + temperature=0.1, max_tokens=16384, timeout=75, ) _raw_original = cal_extract or "" cal_extract = _strip_think(_raw_original) @@ -562,10 +607,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None if matches: cal_extract = matches[-1].group() logger.info(f"[cal-extract] uid={uid.decode() if isinstance(uid, bytes) else uid} folder={_folder} subj={subject[:50]!r} raw_len={len(cal_extract)} orig_len={len(_raw_original)} raw={cal_extract[:800]!r}") - jm = re.search(r'\[.*\]', cal_extract, re.DOTALL) - if jm: + ops = _extract_json_array_from_text(cal_extract) + if ops is not None: try: - ops = json.loads(jm.group()) + _cal_parse_ok = True logger.info(f"[cal-extract] parsed {len(ops)} op(s)") if isinstance(ops, list) and ops: from src.tool_implementations import do_manage_calendar @@ -595,6 +640,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None r = await do_manage_calendar(json.dumps(args), owner=_acct_owner) if r.get("exit_code", 0) == 0: logger.info(f"[cal-extract] Updated event uid={cuid} → {op.get('title')} {op['date']}") + if cuid and cuid not in _cal_event_uids: + _cal_event_uids.append(cuid) _cal_run_count += 1 else: logger.warning(f"[cal-extract] update failed: {r.get('error')}") @@ -675,26 +722,34 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None r = await do_manage_calendar(cal_args, owner=_acct_owner) if r.get("exit_code", 0) == 0: logger.info(f"[cal-extract] Created event: {op['title']} on {op['date']}") + _created_uid = (r.get("uid") or "").strip() + if _created_uid and _created_uid not in _cal_event_uids: + _cal_event_uids.append(_created_uid) _events_created += 1 _cal_run_count += 1 else: logger.warning(f"[cal-extract] create failed: {r.get('error')} args={cal_args[:200]}") except Exception as je: logger.warning(f"[cal-extract] JSON parse failed: {je} on raw={cal_extract[:200]!r}") + else: + logger.warning(f"[cal-extract] no JSON array found on raw={cal_extract[:200]!r}") except Exception as e: logger.warning(f"[cal-extract] Meeting extraction LLM call failed for uid={uid}: {e}") - # Record we processed this email so we don't re-LLM next run + # Record successfully parsed results so we don't re-LLM + # no-op emails. If the model returned no parseable JSON, + # leave it uncached so a future run can retry. try: - _cc = _sql3.connect(SCHEDULED_DB) - _cc.execute( - "INSERT OR REPLACE INTO email_calendar_extractions " - "(message_id, owner, uid, events_created, created_at) VALUES (?, ?, ?, ?, ?)", - (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), - _cal_run_count, datetime.utcnow().isoformat()) - ) - _cc.commit() - _cc.close() - _cal_existing.add(message_id) + if _cal_parse_ok: + _cc = _sql3.connect(SCHEDULED_DB) + _cc.execute( + "INSERT OR REPLACE INTO email_calendar_extractions " + "(message_id, owner, uid, event_uids, events_created, created_at) VALUES (?, ?, ?, ?, ?, ?)", + (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), + json.dumps(_cal_event_uids), _cal_run_count, datetime.utcnow().isoformat()) + ) + _cc.commit() + _cc.close() + _cal_existing.add(message_id) except Exception as ce: logger.debug(f"Could not cache calendar extraction: {ce}") @@ -728,9 +783,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None "temperature": 0, tok_key: 200, } - urg_raw = await llm_call_async( - url=url, model=model, messages=payload["messages"], - temperature=0, max_tokens=200, headers=req_headers, timeout=60, + urg_raw = await task_llm_call_async( + messages=payload["messages"], + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=account_owner or None, + temperature=0, max_tokens=200, timeout=60, ) urg_raw = _strip_think(urg_raw or "") urg_raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", urg_raw, flags=re.MULTILINE).strip() @@ -831,8 +888,13 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None class_sys = ( "Classify the email. Return ONLY a JSON object, no prose, no markdown fences. " "Schema: {\"tags\": [\"tag1\"], \"spam\": false, \"reason\": \"short\"}. " - "Pick 1-2 tags from: work, personal, finance, bills, receipt, travel, " - "newsletter, promo, notification, security, social, shopping, calendar.\n\n" + "Pick 1-3 tags from: work, personal, urgent, action-needed, finance, bills, " + "receipt, legal, travel, newsletter, promo, notification, security, social, " + "shopping, calendar, support.\n\n" + "Use work for professional/company/client/operations messages. " + "Use personal for friends/family/private-life messages. " + "Use urgent for real time-sensitive consequences. " + "Use action-needed when the user likely needs to reply, pay, sign, book, or decide.\n\n" "Set spam=true for ANY of:\n" "- Phishing, scams, chain mail, deceptive offers\n" "- Marketing/promotional blasts (\"special offer\", \"limited time\", discount codes)\n" @@ -849,70 +911,55 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None "If it's a mass-mailed generic update with no personal CTA, mark spam=true even if from a legitimate service. " "Reason should be 5-10 words." ) - tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" - payload = { - "model": model, - "messages": [ + raw_out = await task_llm_call_async( + messages=[ {"role": "system", "content": class_sys}, {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body[:4000]}"}, ], - tok_key: 512, - "temperature": 0.1, - "stream": False, - } - # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature. - if _restricts_temperature(model): - payload.pop("temperature", None) - # to_thread keeps the event loop responsive during the LLM call - resp = await asyncio.to_thread( - _req.post, url, json=payload, headers=req_headers, timeout=120 + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=account_owner or None, + temperature=0.1, max_tokens=512, timeout=120, ) - if not resp.ok: - logger.warning(f"Auto-classify {uid.decode() if isinstance(uid, bytes) else str(uid)} HTTP {resp.status_code}: {resp.text[:200]}") - else: - rdata = resp.json() - m = (rdata.get("choices") or [{}])[0].get("message", {}) - raw_out = (m.get("content") or "").strip() - raw_out = _strip_think(raw_out) - raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip() - jm = re.search(r'\{.*\}', raw_out, re.DOTALL) - parsed = None - if jm: - try: - parsed = json.loads(jm.group(0)) - except Exception: - parsed = None - if parsed is not None: - _ALLOWED_TAGS = {"work","personal","finance","bills","receipt","travel", - "newsletter","marketing","notification","security","social", - "shopping","calendar"} - raw_tags = parsed.get("tags") or [] - if isinstance(raw_tags, str): - raw_tags = [raw_tags] - tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)] - tags = ["marketing" if t == "promo" else t for t in tags] - tags = [t for t in tags if t in _ALLOWED_TAGS][:2] - is_spam = bool(parsed.get("spam")) - spam_reason = str(parsed.get("reason") or "")[:200] + raw_out = _strip_think((raw_out or "").strip()) + raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip() + jm = re.search(r'\{.*\}', raw_out, re.DOTALL) + parsed = None + if jm: + try: + parsed = json.loads(jm.group(0)) + except Exception: + parsed = None + if parsed is not None: + _ALLOWED_TAGS = {"work","personal","urgent","action-needed","finance","bills", + "receipt","legal","travel","newsletter","marketing","notification", + "security","social","shopping","calendar","support"} + raw_tags = parsed.get("tags") or [] + if isinstance(raw_tags, str): + raw_tags = [raw_tags] + tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)] + tags = ["marketing" if t == "promo" else t for t in tags] + tags = [t for t in tags if t in _ALLOWED_TAGS][:3] + is_spam = bool(parsed.get("spam")) + spam_reason = str(parsed.get("reason") or "")[:200] - moved_to = "" - if is_spam and auto_spam and spam_folder: - if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner): - moved_to = spam_folder - logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}") + moved_to = "" + if is_spam and auto_spam and spam_folder: + if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner): + moved_to = spam_folder + logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}") - _c = _sql3.connect(SCHEDULED_DB) - _c.execute(""" - INSERT OR REPLACE INTO email_tags - (message_id, owner, uid, folder, subject, sender, tags, spam_verdict, - spam_reason, moved_to, model_used, created_at) - VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?, ?, ?) - """, (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), subject, sender, - json.dumps(tags), 1 if is_spam else 0, - spam_reason, moved_to, model, datetime.utcnow().isoformat())) - _c.commit() - _c.close() - _tag_existing.add(message_id) + _c = _sql3.connect(SCHEDULED_DB) + _c.execute(""" + INSERT OR REPLACE INTO email_tags + (message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict, + spam_reason, moved_to, model_used, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (message_id, account_owner or "", account_id or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, + json.dumps(tags), 1 if is_spam else 0, + spam_reason, moved_to, model, datetime.utcnow().isoformat())) + _c.commit() + _c.close() + _tag_existing.add(message_id) except Exception as e: logger.warning(f"Auto-classify {uid} failed: {e}") diff --git a/routes/email_routes.py b/routes/email_routes.py index 77be0cdeb..f90782c20 100644 --- a/routes/email_routes.py +++ b/routes/email_routes.py @@ -108,6 +108,71 @@ def _email_tag_owner_clause(account_id: str | None, owner: str = "") -> tuple[st return f"(owner IN ({placeholders}) OR owner IS NULL)", aliases +def _email_tag_account_clause(account_id: str | None) -> tuple[str, list[str]]: + account = (account_id or "").strip() + if account: + return "(account_id=? OR account_id='' OR account_id IS NULL)", [account] + # No explicit account means the caller is using the default/all-account + # view. Keep the owner clause as the boundary, but do not hide tags that + # were written under a concrete account id for the same message. + return "1=1", [] + + +_VISIBLE_EMAIL_TAGS = {"urgent", "reply-soon", "action-needed", "calendar", "bills", "receipt", "travel"} +_DONE_RESPONSE_TAGS = {"urgent", "reply-soon", "action-needed"} + + +def _sanitize_visible_email_tags(tags, *, is_answered: bool = False) -> list[str]: + out = [] + for tag in tags if isinstance(tags, list) else []: + tag = str(tag or "").strip().lower().replace("_", "-") + if tag == "promo": + tag = "marketing" + if tag not in _VISIBLE_EMAIL_TAGS: + continue + if is_answered and tag in _DONE_RESPONSE_TAGS: + continue + if tag not in out: + out.append(tag) + return out + + +def _hide_unlinked_calendar_tags(emails: list[dict]) -> None: + for e in emails or []: + if not isinstance(e.get("tags"), list): + continue + if "calendar" in e.get("tags", []) and not e.get("calendar_event_uids"): + e["tags"] = [t for t in e.get("tags", []) if t != "calendar"] + + +def _clear_done_response_tags(owner: str, account_id: str | None, folder: str, uid: str) -> None: + try: + conn = _sql3.connect(SCHEDULED_DB) + owner_clause, owner_params = _email_tag_owner_clause(account_id, owner) + account_clause, account_params = _email_tag_account_clause(account_id) + rows = conn.execute( + f"SELECT rowid, tags FROM email_tags WHERE folder=? AND uid=? AND {owner_clause} AND {account_clause}", + [folder, str(uid), *owner_params, *account_params], + ).fetchall() + for rowid, tags_raw in rows: + try: + tags = json.loads(tags_raw or "[]") + except Exception: + tags = [] + if not isinstance(tags, list): + tags = [] + kept = [ + t for t in tags + if str(t).strip().lower().replace("_", "-") not in _DONE_RESPONSE_TAGS + ] + if kept != tags: + conn.execute("UPDATE email_tags SET tags=? WHERE rowid=?", (json.dumps(kept), rowid)) + conn.commit() + conn.close() + except Exception as e: + logger.debug(f"clear done response tags skipped: {e}") + + def _record_email_received_events(owner: str, account_id: str | None, folder: str, emails: list[dict]): """Baseline inbox messages, then fire `email_received` for new arrivals.""" if not owner or (folder or "INBOX").upper() != "INBOX" or not emails: @@ -304,6 +369,433 @@ def _group_uid_fetch_records(msg_data) -> list: return grouped +def _account_cache_key(account_id: str | None, owner: str = "") -> str: + return (account_id or "default").strip() or f"default:{owner or ''}" + + +def _parse_email_list_record(meta_b: bytes, raw_header: bytes | None) -> dict | None: + try: + meta = meta_b.decode(errors="replace") + uid_num = _uid_from_fetch_meta(meta_b) + if not uid_num or not raw_header: + return None + flag_m = re.search(r'FLAGS \(([^)]*)\)', meta) + flags = flag_m.group(1) if flag_m else "" + size_m = re.search(r'RFC822\.SIZE (\d+)', meta) + size = int(size_m.group(1)) if size_m else 0 + msg = email_mod.message_from_bytes(raw_header) + subject = _decode_header(msg.get("Subject", "(no subject)")) + sender = _decode_header(msg.get("From", "unknown")) + date_str = msg.get("Date", "") + message_id = (msg.get("Message-ID", "") or "").strip() + sender_name, sender_addr = email.utils.parseaddr(sender) + to_str = _decode_header(msg.get("To", "")) + cc_str = _decode_header(msg.get("Cc", "")) + parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None + if parsed_date and parsed_date.tzinfo is None: + from datetime import timezone as _tz + parsed_date = parsed_date.replace(tzinfo=_tz.utc) + iso_date = parsed_date.isoformat() if parsed_date else "" + date_epoch = parsed_date.timestamp() if parsed_date else 0.0 + ct = msg.get("Content-Type", "") + has_attachments = "multipart/mixed" in ct.lower() or "multipart/related" in ct.lower() + return { + "uid": uid_num, + "message_id": message_id, + "subject": subject, + "from_name": sender_name or sender_addr, + "from_address": sender_addr, + "to": to_str, + "cc": cc_str, + "date": iso_date, + "date_display": date_str, + "date_epoch": date_epoch, + "size": size, + "is_read": "\\Seen" in flags, + "is_answered": "\\Answered" in flags, + "is_flagged": "\\Flagged" in flags, + "flags": flags, + "has_attachments": has_attachments, + } + except Exception as e: + logger.warning(f"Error parsing email index entry: {e}") + return None + + +def _email_index_rows(owner: str, account_id: str | None, folder: str, uids: list[str]) -> dict[str, dict]: + if not uids: + return {} + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + placeholders = ",".join("?" * len(uids)) + rows = conn.execute( + f""" + SELECT uid, message_id, subject, from_name, from_address, to_text, cc_text, + date_iso, date_display, date_epoch, size, flags, has_attachments + FROM email_message_index + WHERE owner=? AND account_key=? AND folder=? AND uid IN ({placeholders}) + """, + [owner or "", _account_cache_key(account_id, owner), folder, *uids], + ).fetchall() + finally: + conn.close() + except Exception as e: + logger.debug(f"email index read skipped: {e}") + return {} + out: dict[str, dict] = {} + for row in rows: + uid, message_id, subject, from_name, from_address, to_text, cc_text, date_iso, date_display, date_epoch, size, flags, has_attachments = row + flags = flags or "" + out[str(uid)] = { + "uid": str(uid), + "message_id": (message_id or "").strip(), + "subject": subject or "(no subject)", + "from_name": from_name or from_address or "", + "from_address": from_address or "", + "to": to_text or "", + "cc": cc_text or "", + "date": date_iso or "", + "date_display": date_display or "", + "date_epoch": float(date_epoch or 0), + "size": int(size or 0), + "is_read": "\\Seen" in flags, + "is_answered": "\\Answered" in flags, + "is_flagged": "\\Flagged" in flags, + "flags": flags, + "has_attachments": bool(has_attachments), + } + return out + + +def _email_index_search(owner: str, account_id: str | None, folder: str, query: str, limit: int) -> tuple[list[dict], int, str | None]: + q = (query or "").strip() + if not q: + return [], 0, None + limit = max(1, min(int(limit or 50), 200)) + account_key = _account_cache_key(account_id, owner) + like = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" + folder_clause = "" + params: list = [owner or "", account_key] + # Searching from INBOX should feel global for Gmail-style accounts, + # because users expect archived/labelled mail to show up too. The + # local index only contains folders that have been warmed/listed, so + # this remains a best-effort fast path; IMAP is still the fallback. + if (folder or "").upper() != "INBOX": + folder_clause = "AND folder=?" + params.append(folder) + params.extend([like, like, like, like, like]) + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + total_row = conn.execute( + f""" + SELECT COUNT(*), MAX(updated_at) + FROM email_message_index + WHERE owner=? AND account_key=? {folder_clause} + AND ( + subject LIKE ? ESCAPE '\\' OR + from_name LIKE ? ESCAPE '\\' OR + from_address LIKE ? ESCAPE '\\' OR + to_text LIKE ? ESCAPE '\\' OR + cc_text LIKE ? ESCAPE '\\' + ) + """, + params, + ).fetchone() + total = int((total_row or [0])[0] or 0) + if not total: + return [], 0, (total_row or [None, None])[1] + rows = conn.execute( + f""" + SELECT uid, message_id, subject, from_name, from_address, to_text, cc_text, + date_iso, date_display, date_epoch, size, flags, has_attachments, + folder + FROM email_message_index + WHERE owner=? AND account_key=? {folder_clause} + AND ( + subject LIKE ? ESCAPE '\\' OR + from_name LIKE ? ESCAPE '\\' OR + from_address LIKE ? ESCAPE '\\' OR + to_text LIKE ? ESCAPE '\\' OR + cc_text LIKE ? ESCAPE '\\' + ) + ORDER BY date_epoch DESC + LIMIT ? + """, + [*params, limit], + ).fetchall() + finally: + conn.close() + except Exception: + logger.debug("email index search skipped", exc_info=True) + return [], 0, None + + emails: list[dict] = [] + for row in rows: + uid, message_id, subject, from_name, from_address, to_text, cc_text, date_iso, date_display, date_epoch, size, flags, has_attachments, row_folder = row + flags = flags or "" + emails.append({ + "uid": str(uid), + "message_id": (message_id or "").strip(), + "subject": subject or "(no subject)", + "from_name": from_name or from_address or "", + "from_address": from_address or "", + "to": to_text or "", + "cc": cc_text or "", + "date": date_iso or "", + "date_display": date_display or "", + "date_epoch": float(date_epoch or 0), + "size": int(size or 0), + "is_read": "\\Seen" in flags, + "is_answered": "\\Answered" in flags, + "is_flagged": "\\Flagged" in flags, + "flags": flags, + "has_attachments": bool(has_attachments), + "folder": row_folder or folder, + }) + return emails, total, (total_row or [None, None])[1] + + +def _email_index_upsert(owner: str, account_id: str | None, folder: str, emails: list[dict]): + if not emails: + return + now = datetime.utcnow().isoformat() + "Z" + rows = [] + for e in emails: + uid = str(e.get("uid") or "").strip() + if not uid: + continue + rows.append(( + owner or "", + _account_cache_key(account_id, owner), + folder, + uid, + (e.get("message_id") or "").strip(), + e.get("subject") or "", + e.get("from_name") or "", + e.get("from_address") or "", + e.get("to") or "", + e.get("cc") or "", + e.get("date") or "", + e.get("date_display") or "", + float(e.get("date_epoch") or 0), + int(e.get("size") or 0), + e.get("flags") or "", + 1 if e.get("has_attachments") else 0, + now, + )) + if not rows: + return + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + conn.executemany( + """ + INSERT INTO email_message_index + (owner, account_key, folder, uid, message_id, subject, from_name, + from_address, to_text, cc_text, date_iso, date_display, date_epoch, + size, flags, has_attachments, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(owner, account_key, folder, uid) DO UPDATE SET + message_id=excluded.message_id, + subject=excluded.subject, + from_name=excluded.from_name, + from_address=excluded.from_address, + to_text=excluded.to_text, + cc_text=excluded.cc_text, + date_iso=excluded.date_iso, + date_display=excluded.date_display, + date_epoch=excluded.date_epoch, + size=excluded.size, + flags=excluded.flags, + has_attachments=excluded.has_attachments, + updated_at=excluded.updated_at + """, + rows, + ) + conn.commit() + finally: + conn.close() + except Exception as e: + logger.debug(f"email index write skipped: {e}") + + +def _email_index_update_flags(owner: str, account_id: str | None, folder: str, uid: str, flag: str, add: bool): + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + row = conn.execute( + "SELECT flags FROM email_message_index WHERE owner=? AND account_key=? AND folder=? AND uid=?", + (owner or "", _account_cache_key(account_id, owner), folder, str(uid)), + ).fetchone() + if not row: + return + parts = {p for p in (row[0] or "").split() if p} + if add: + parts.add(flag) + else: + parts.discard(flag) + conn.execute( + "UPDATE email_message_index SET flags=?, updated_at=? WHERE owner=? AND account_key=? AND folder=? AND uid=?", + (" ".join(sorted(parts)), datetime.utcnow().isoformat() + "Z", owner or "", _account_cache_key(account_id, owner), folder, str(uid)), + ) + conn.commit() + finally: + conn.close() + except Exception: + logger.debug("email index flag update skipped", exc_info=True) + + +def _email_index_delete(owner: str, account_id: str | None, folder: str | None, uid: str): + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + if folder: + conn.execute( + "DELETE FROM email_message_index WHERE owner=? AND account_key=? AND folder=? AND uid=?", + (owner or "", _account_cache_key(account_id, owner), folder, str(uid)), + ) + else: + conn.execute( + "DELETE FROM email_message_index WHERE owner=? AND account_key=? AND uid=?", + (owner or "", _account_cache_key(account_id, owner), str(uid)), + ) + conn.commit() + finally: + conn.close() + except Exception: + logger.debug("email index delete skipped", exc_info=True) + + +def _email_preview_cache_get(owner: str, account_id: str | None, folder: str, uid: str) -> dict | None: + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + row = conn.execute( + """ + SELECT payload_json, updated_at + FROM email_body_preview_cache + WHERE owner=? AND account_key=? AND folder=? AND uid=? + """, + (owner or "", _account_cache_key(account_id, owner), folder, str(uid)), + ).fetchone() + finally: + conn.close() + if not row: + return None + payload = json.loads(row[0] or "{}") + if isinstance(payload, dict): + payload.setdefault("sync", {}) + payload["sync"].update({"source": "preview_cache", "updated_at": row[1]}) + return payload + except Exception: + logger.debug("email preview cache read skipped", exc_info=True) + return None + + +def _email_preview_cache_put(owner: str, account_id: str | None, folder: str, uid: str, payload: dict): + if not payload: + return + try: + now = datetime.utcnow().isoformat() + "Z" + message_id = (payload.get("message_id") or "").strip() + stored = dict(payload) + stored["sync"] = {"source": "preview_cache", "updated_at": now} + conn = _sql3.connect(SCHEDULED_DB) + try: + conn.execute( + """ + INSERT INTO email_body_preview_cache + (owner, account_key, folder, uid, message_id, payload_json, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(owner, account_key, folder, uid) DO UPDATE SET + message_id=excluded.message_id, + payload_json=excluded.payload_json, + updated_at=excluded.updated_at + """, + ( + owner or "", + _account_cache_key(account_id, owner), + folder, + str(uid), + message_id, + json.dumps(stored, ensure_ascii=False), + now, + ), + ) + conn.commit() + finally: + conn.close() + except Exception: + logger.debug("email preview cache write skipped", exc_info=True) + + +def _email_attachment_meta_cache_get(owner: str, account_id: str | None, folder: str, uid: str) -> list[dict] | None: + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + row = conn.execute( + """ + SELECT attachments_json + FROM email_attachment_metadata_cache + WHERE owner=? AND account_key=? AND folder=? AND uid=? + """, + (owner or "", _account_cache_key(account_id, owner), folder, str(uid)), + ).fetchone() + if not row: + row = conn.execute( + """ + SELECT attachments_json + FROM email_attachment_metadata_cache + WHERE owner=? AND folder=? AND uid=? + ORDER BY updated_at DESC + LIMIT 1 + """, + (owner or "", folder, str(uid)), + ).fetchone() + finally: + conn.close() + if not row: + return None + data = json.loads(row[0] or "[]") + return data if isinstance(data, list) else [] + except Exception: + logger.debug("email attachment metadata cache read skipped", exc_info=True) + return None + + +def _email_attachment_meta_cache_put(owner: str, account_id: str | None, folder: str, uid: str, message_id: str, attachments: list[dict]): + try: + conn = _sql3.connect(SCHEDULED_DB) + try: + conn.execute( + """ + INSERT INTO email_attachment_metadata_cache + (owner, account_key, folder, uid, message_id, attachments_json, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(owner, account_key, folder, uid) DO UPDATE SET + message_id=excluded.message_id, + attachments_json=excluded.attachments_json, + updated_at=excluded.updated_at + """, + ( + owner or "", + _account_cache_key(account_id, owner), + folder, + str(uid), + (message_id or "").strip(), + json.dumps(attachments or [], ensure_ascii=False), + datetime.utcnow().isoformat() + "Z", + ), + ) + conn.commit() + finally: + conn.close() + except Exception: + logger.debug("email attachment metadata cache write skipped", exc_info=True) + + def _smtp_ready(cfg: dict) -> bool: if not cfg.get("smtp_host") or not cfg.get("smtp_user"): return False @@ -550,14 +1042,16 @@ def setup_email_routes(): import threading as _threading _LIST_CACHE = {} # key → (expires_at, response_dict) - _LIST_TTL = 8.0 + _LIST_TTL = 45.0 + _FOLDER_CACHE = {} # (account_id, owner) -> (expires_at, response_dict) + _FOLDER_TTL = 5 * 60.0 _READ_CACHE = {} # key → (expires_at, response_dict) _READ_TTL = 30 * 60.0 _IMAP_POOL = {} # account_id → (conn, last_used_at) _IMAP_IDLE_MAX = 60.0 _WARMING_READS = set() - _WARM_READ_LIMIT = 1 - _WARM_MAX_BYTES = 128 * 1024 + _WARM_READ_LIMIT = 2 + _WARM_MAX_BYTES = 192 * 1024 _WARM_RECENT_SECONDS = 7 * 24 * 60 * 60 _pool_lock = _threading.Lock() @@ -629,6 +1123,22 @@ def setup_email_routes(): for k in list(_LIST_CACHE.keys())[:-32]: _LIST_CACHE.pop(k, None) + def _folder_cache_get(account_id, owner): + key = (account_id or "", owner or "") + v = _FOLDER_CACHE.get(key) + if not v: + return None + if v[0] < _time.monotonic(): + _FOLDER_CACHE.pop(key, None) + return None + return v[1] + + def _folder_cache_put(account_id, owner, value): + _FOLDER_CACHE[(account_id or "", owner or "")] = (_time.monotonic() + _FOLDER_TTL, value) + if len(_FOLDER_CACHE) > 32: + for k in list(_FOLDER_CACHE.keys())[:-16]: + _FOLDER_CACHE.pop(k, None) + def _invalidate_list_cache(account_id=None, folder=None): """Drop list cache entries that the caller's mutation may have stale-ed. @@ -646,6 +1156,28 @@ def setup_email_routes(): (folder is None or k_folder == folder): _LIST_CACHE.pop(k, None) + def _update_list_cache_seen(account_id, folder, uid, seen: bool): + uid_s = str(uid) + for key, (expires_at, value) in list(_LIST_CACHE.items()): + if key[0] != (account_id or "") or key[1] != folder: + continue + emails = list((value or {}).get("emails") or []) + changed = False + if seen and key[2] == "unread": + kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] + if len(kept) != len(emails): + value = dict(value) + value["emails"] = kept + value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) + changed = True + else: + for e in emails: + if str((e or {}).get("uid") or "") == uid_s: + e["is_read"] = bool(seen) + changed = True + if changed: + _LIST_CACHE[key] = (expires_at, value) + def _read_cache_get(key): v = _READ_CACHE.get(key) if not v: return None @@ -677,6 +1209,144 @@ def setup_email_routes(): _POOL_HOOKS["connect"] = _pooled_connect _POOL_HOOKS["release"] = _pooled_release + def _fixture_email_file() -> Path: + return Path(DATA_DIR) / "fixture_email_messages.json" + + def _fixture_email_enabled() -> bool: + return _fixture_email_file().exists() + + def _fixture_email_rows(owner: str) -> list[dict]: + path = _fixture_email_file() + if not path.exists(): + return [] + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except Exception: + logger.debug("fixture email load failed", exc_info=True) + return [] + rows = raw.get("messages") if isinstance(raw, dict) else raw + out = [] + for i, row in enumerate(rows if isinstance(rows, list) else [], start=1): + if not isinstance(row, dict): + continue + row_owner = str(row.get("owner") or "").strip() + if owner and row_owner and row_owner != owner: + continue + out.append(_fixture_email_record(row, i, owner)) + out.sort(key=lambda e: e.get("date_epoch") or 0, reverse=True) + return out + + def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict: + sender = str(row.get("from") or "Fixture Sender ") + sender_name, sender_addr = email.utils.parseaddr(sender) + raw_date = str(row.get("date") or "") + parsed_date = None + if raw_date: + try: + parsed_date = datetime.fromisoformat(raw_date.replace("Z", "+00:00")) + except Exception: + try: + parsed_date = email.utils.parsedate_to_datetime(raw_date) + except Exception: + parsed_date = None + iso_date = parsed_date.isoformat() if parsed_date else raw_date + date_epoch = parsed_date.timestamp() if parsed_date else 0.0 + subject = str(row.get("subject") or "(no subject)") + body = str(row.get("body") or "") + uid = str(uid_num) + owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default") + return { + "uid": uid, + "message_id": f"", + "subject": subject, + "from_name": sender_name or sender_addr or sender, + "from_address": sender_addr, + "to": owner or "", + "cc": "", + "date": iso_date, + "date_display": raw_date, + "date_epoch": date_epoch, + "size": len(body.encode("utf-8")), + "is_read": False, + "is_answered": False, + "is_flagged": False, + "flags": "", + "has_attachments": False, + "folder": "INBOX", + "_fixture_body": body, + } + + def _fixture_email_list(folder: str, limit: int, offset: int, filter_: str, from_addr: str | None, owner: str) -> dict | None: + if not _fixture_email_enabled(): + return None + if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}: + return {"emails": [], "total": 0, "folder": folder, "sync": {"source": "fixture"}} + rows = _fixture_email_rows(owner) + if from_addr: + needle = from_addr.strip().lower() + rows = [ + e for e in rows + if needle in (e.get("from_address") or "").lower() + or needle in (e.get("from_name") or "").lower() + ] + if filter_ in {"unread", "unanswered", "undone", "all", "", None}: + pass + elif filter_ in {"favorites", "reminders"} or str(filter_).startswith("tag:"): + rows = [] + else: + pass + total = len(rows) + start = max(0, int(offset or 0)) + stop = start + max(1, min(int(limit or 50), 200)) + visible = [] + for e in rows[start:stop]: + item = dict(e) + item.pop("_fixture_body", None) + visible.append(item) + return { + "emails": visible, + "total": total, + "folder": folder, + "sync": {"source": "fixture", "updated_at": datetime.utcnow().isoformat() + "Z"}, + } + + def _fixture_email_read(uid: str, folder: str, owner: str) -> dict | None: + if not _fixture_email_enabled(): + return None + if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}: + return {"error": f"Email UID {uid} not found"} + for e in _fixture_email_rows(owner): + if str(e.get("uid")) != str(uid): + continue + body = e.get("_fixture_body") or "" + body_html = "

" + html.escape(body).replace("\n", "
") + "

" + return { + "uid": str(uid), + "folder": folder, + "message_id": e.get("message_id") or "", + "subject": e.get("subject") or "", + "from_name": e.get("from_name") or "", + "from_address": e.get("from_address") or "", + "to": e.get("to") or owner or "", + "cc": "", + "date": e.get("date") or "", + "in_reply_to": "", + "references": "", + "body": body, + "body_html": body_html, + "attachments": [], + "attachments_deferred": False, + "related_attachments": [], + "attachment_version": EMAIL_READ_ATTACHMENT_VERSION, + "cached_summary": None, + "cached_ai_reply": None, + "boundaries": None, + "thread_turns": None, + "sender_signature": None, + "sync": {"source": "fixture"}, + } + return {"error": f"Email UID {uid} not found"} + def _list_emails_sync(folder, limit, offset, filter_, account_id, from_addr=None, has_attachments_only=False, owner=""): """Sync IMAP work — call from async handler via asyncio.to_thread so it doesn't block the event loop. @@ -691,8 +1361,10 @@ def setup_email_routes(): the fallback config lookup is scoped to this user's accounts only. """ conn = None + conn_ok = False try: - conn = _imap_connect(account_id, owner=owner) + conn, _reused_conn = _pooled_connect(account_id, owner=owner) + conn_ok = True select_status, _ = conn.select(_q(folder), readonly=True) if select_status != "OK": return {"emails": [], "total": 0, "folder": folder, "error": f"Folder not found: {folder}"} @@ -745,6 +1417,7 @@ def setup_email_routes(): import sqlite3 as _sql3t _ct = _sql3t.connect(SCHEDULED_DB) _owner_clause, _owner_params = _email_tag_owner_clause(account_id, owner) + _account_clause, _account_params = _email_tag_account_clause(account_id) # SECURITY: owner-scope the lookup (review C2/H8). Without # this, user A's `tag:urgent` filter would surface UIDs # written by user B and IMAP would return whatever @@ -756,8 +1429,8 @@ def setup_email_routes(): rows_t = _ct.execute( "SELECT message_id, uid FROM email_tags " "WHERE folder=? AND spam_verdict=1 " - f"AND {_owner_clause}", - (folder, *_owner_params), + f"AND {_owner_clause} AND {_account_clause}", + (folder, *_owner_params, *_account_params), ).fetchall() for mid, uid in rows_t: if mid: @@ -768,17 +1441,40 @@ def setup_email_routes(): rows_t = _ct.execute( "SELECT message_id, uid, tags FROM email_tags " "WHERE folder=? AND tags IS NOT NULL AND tags != '' " - f"AND {_owner_clause}", - (folder, *_owner_params), + f"AND {_owner_clause} AND {_account_clause}", + (folder, *_owner_params, *_account_params), ).fetchall() + _idx_flags_by_uid = {} + _idx_flags_by_mid = {} + try: + _uid_vals = [str(r[1]).strip() for r in rows_t if r[1]] + _mid_vals = [str(r[0]).strip() for r in rows_t if r[0]] + _idx_clauses = [] + _idx_params = [owner or "", _account_cache_key(account_id, owner), folder] + if _uid_vals: + _idx_clauses.append("uid IN (" + ",".join("?" * len(_uid_vals)) + ")") + _idx_params.extend(_uid_vals) + if _mid_vals: + _idx_clauses.append("message_id IN (" + ",".join("?" * len(_mid_vals)) + ")") + _idx_params.extend(_mid_vals) + if _idx_clauses: + for _uid_i, _mid_i, _flags_i in _ct.execute( + "SELECT uid, message_id, flags FROM email_message_index " + "WHERE owner=? AND account_key=? AND folder=? AND (" + " OR ".join(_idx_clauses) + ")", + _idx_params, + ).fetchall(): + if _uid_i: + _idx_flags_by_uid[str(_uid_i)] = _flags_i or "" + if _mid_i: + _idx_flags_by_mid[str(_mid_i).strip()] = _flags_i or "" + except Exception as _idx_e: + logger.debug(f"tag filter index flag lookup skipped: {_idx_e}") for r in rows_t: try: tg = json.loads(r[2] or "[]") - wanted = {_tag_name} - if _tag_name == "marketing": - wanted.add("promo") - row_tags = {str(t).strip().lower().replace("_", "-") for t in tg} if isinstance(tg, list) else set() - if wanted.intersection(row_tags): + flags = _idx_flags_by_mid.get(str(r[0] or "").strip()) or _idx_flags_by_uid.get(str(r[1] or "").strip()) or "" + row_tags = set(_sanitize_visible_email_tags(tg, is_answered="\\Answered" in flags)) + if _tag_name in row_tags: if r[0]: _tag_message_ids.append(str(r[0]).strip()) elif r[1]: @@ -789,7 +1485,6 @@ def setup_email_routes(): except Exception as _te: logger.warning(f"tag filter lookup failed: {_te}") if not _tag_message_ids and not _tag_seq_fallback: - conn.logout() return {"emails": [], "total": 0, "folder": folder} # Prefer stable Message-ID rows. Older tag rows may have only # numeric ids; those were sequence numbers historically, but @@ -807,7 +1502,6 @@ def setup_email_routes(): if _uid: _uids.add(str(_uid).encode()) if not _uids: - conn.logout() return {"emails": [], "total": 0, "folder": folder} data = [b" ".join(sorted(_uids, key=lambda x: int(x) if str(x, "ascii", "ignore").isdigit() else 0))] status = "OK" @@ -817,7 +1511,6 @@ def setup_email_routes(): status, data = _imap_uid_search(conn, "ALL") if status != "OK" or not data[0]: - conn.logout() return {"emails": [], "total": 0, "folder": folder} uid_list = data[0].split() @@ -842,10 +1535,11 @@ def setup_email_routes(): if _uid_strs: placeholders = ",".join("?" * len(_uid_strs)) _owner_clause, _owner_params = _email_tag_owner_clause(account_id, owner) + _account_clause, _account_params = _email_tag_account_clause(account_id) rows = _c.execute( f"SELECT uid, tags, spam_verdict FROM email_tags " - f"WHERE folder=? AND {_owner_clause} AND uid IN ({placeholders})", - [folder, *_owner_params, *_uid_strs], + f"WHERE folder=? AND {_owner_clause} AND {_account_clause} AND uid IN ({placeholders})", + [folder, *_owner_params, *_account_params, *_uid_strs], ).fetchall() for r in rows: try: @@ -853,7 +1547,7 @@ def setup_email_routes(): except Exception: tg = [] if isinstance(tg, list): - tg = ["marketing" if str(t).strip().lower().replace("_", "-") == "promo" else t for t in tg] + tg = _sanitize_visible_email_tags(tg) _tag_by_uid[r[0]] = {"tags": tg, "spam": bool(r[2])} _c.close() except Exception as e: @@ -865,12 +1559,18 @@ def setup_email_routes(): # batched form trades a slightly bigger response for one round-trip. emails = [] if uid_list: - fetch_set = b",".join(uid_list) - try: - status, msg_data = _imap_uid_fetch(conn, fetch_set, "(UID FLAGS RFC822.HEADER RFC822.SIZE)") - except Exception as e: - logger.warning(f"Batch fetch failed, falling back to per-UID: {e}") - status, msg_data = "NO", [] + uid_order = [u.decode(errors="ignore") if isinstance(u, bytes) else str(u) for u in uid_list] + cached_rows = _email_index_rows(owner, account_id, folder, uid_order) + missing_uids = [u for u in uid_order if u and u not in cached_rows] + fetched_emails = [] + status, msg_data = "OK", [] + if missing_uids: + fetch_set = ",".join(missing_uids).encode() + try: + status, msg_data = _imap_uid_fetch(conn, fetch_set, "(UID FLAGS RFC822.HEADER RFC822.SIZE)") + except Exception as e: + logger.warning(f"Batch fetch failed, falling back to per-UID: {e}") + status, msg_data = "NO", [] # Group the batched response into per-message (meta, payload) # records. Bare bytes parts must be kept: Gmail returns FLAGS # after the header literal as a bare element, and dropping it @@ -878,7 +1578,6 @@ def setup_email_routes(): grouped = _group_uid_fetch_records(msg_data) if status != "OK" and not grouped: - conn.logout() return {"emails": [], "total": total, "folder": folder, "offset": offset} _tag_by_message_id = {} @@ -894,12 +1593,13 @@ def setup_email_routes(): import sqlite3 as _sql3m _cm = _sql3m.connect(SCHEDULED_DB) _owner_clause_m, _owner_params_m = _email_tag_owner_clause(account_id, owner) + _account_clause_m, _account_params_m = _email_tag_account_clause(account_id) _mid_ph = ",".join("?" * len(header_ids)) rows_m = _cm.execute( f"SELECT message_id, tags, spam_verdict FROM email_tags " - f"WHERE folder=? AND {_owner_clause_m} " + f"WHERE folder=? AND {_owner_clause_m} AND {_account_clause_m} " f"AND message_id IN ({_mid_ph})", - [folder, *_owner_params_m, *header_ids], + [folder, *_owner_params_m, *_account_params_m, *header_ids], ).fetchall() _cm.close() for mid, tags_raw, spam_raw in rows_m: @@ -908,7 +1608,7 @@ def setup_email_routes(): except Exception: tags = [] if isinstance(tags, list): - tags = ["marketing" if str(t).strip().lower().replace("_", "-") == "promo" else t for t in tags] + tags = _sanitize_visible_email_tags(tags) _tag_by_message_id[(mid or "").strip()] = { "tags": tags if isinstance(tags, list) else [], "spam": bool(spam_raw), @@ -917,72 +1617,105 @@ def setup_email_routes(): logger.warning(f"Message-ID tag preload failed: {e}") for meta_b, raw_header in grouped: - try: - meta = meta_b.decode(errors="replace") - uid_num = _uid_from_fetch_meta(meta_b) - if not uid_num: - continue - flag_m = re.search(r'FLAGS \(([^)]*)\)', meta) - flags = flag_m.group(1) if flag_m else "" - size_m = re.search(r'RFC822\.SIZE (\d+)', meta) - size = int(size_m.group(1)) if size_m else 0 - if not raw_header: - continue - - msg = email_mod.message_from_bytes(raw_header) - subject = _decode_header(msg.get("Subject", "(no subject)")) - sender = _decode_header(msg.get("From", "unknown")) - date_str = msg.get("Date", "") - message_id = msg.get("Message-ID", "") - sender_name, sender_addr = email.utils.parseaddr(sender) - # To/Cc — needed for the from-sender sidebar's - # multi-tag filter ("emails involving ALL these - # people"). Decoded raw strings; client splits. - to_str = _decode_header(msg.get("To", "")) - cc_str = _decode_header(msg.get("Cc", "")) - parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None - # Normalise tz-naive parses to UTC so timestamp() is - # deterministic across hosts. - if parsed_date and parsed_date.tzinfo is None: - from datetime import timezone as _tz - parsed_date = parsed_date.replace(tzinfo=_tz.utc) - iso_date = parsed_date.isoformat() if parsed_date else "" - date_epoch = parsed_date.timestamp() if parsed_date else 0.0 - is_read = "\\Seen" in flags - is_answered = "\\Answered" in flags - is_flagged = "\\Flagged" in flags - ct = msg.get("Content-Type", "") - has_attachments = "multipart/mixed" in ct.lower() or "multipart/related" in ct.lower() - tag_entry = _tag_by_message_id.get(message_id.strip()) or _tag_by_uid.get(uid_num, {}) - emails.append({ - "uid": uid_num, - "message_id": message_id.strip(), - "subject": subject, - "from_name": sender_name or sender_addr, - "from_address": sender_addr, - "to": to_str, - "cc": cc_str, - "date": iso_date, - "date_display": date_str, - "date_epoch": date_epoch, - "size": size, - "is_read": is_read, - "is_answered": is_answered, - "is_flagged": is_flagged, - "flags": flags, - "has_attachments": has_attachments, - "tags": tag_entry.get("tags", []), - "is_spam_verdict": tag_entry.get("spam", False), - }) - except Exception as e: - logger.warning(f"Error parsing batched email entry: {e}") + parsed = _parse_email_list_record(meta_b, raw_header) + if parsed: + fetched_emails.append(parsed) + cached_rows[parsed["uid"]] = parsed + _email_index_upsert(owner, account_id, folder, fetched_emails) + for uid_num in uid_order: + row = cached_rows.get(uid_num) + if not row: continue + tag_entry = _tag_by_message_id.get((row.get("message_id") or "").strip()) or _tag_by_uid.get(uid_num, {}) + item = dict(row) + item["tags"] = _sanitize_visible_email_tags(tag_entry.get("tags", []), is_answered=bool(item.get("is_answered"))) + item["is_spam_verdict"] = tag_entry.get("spam", False) + emails.append(item) # IMAP returns batched results in seq-set order, not the # newest-first order we want. Sort by the parsed UTC epoch # so cross-timezone dates compare chronologically (ISO-string # sort had `+02:00` beating `+00:00` at the same local time). emails.sort(key=lambda x: x.get("date_epoch") or 0.0, reverse=True) + if emails: + try: + import sqlite3 as _sql3i + _ci = _sql3i.connect(SCHEDULED_DB) + _owner_clause_i, _owner_params_i = _email_tag_owner_clause(account_id, owner) + _account_clause_i, _account_params_i = _email_tag_account_clause(account_id) + uid_vals = [str(e.get("uid") or "") for e in emails if e.get("uid")] + mid_vals = [(e.get("message_id") or "").strip() for e in emails if e.get("message_id")] + clauses = [] + params = [folder, *_owner_params_i, *_account_params_i] + if uid_vals: + clauses.append("uid IN (" + ",".join("?" * len(uid_vals)) + ")") + params.extend(uid_vals) + if mid_vals: + clauses.append("message_id IN (" + ",".join("?" * len(mid_vals)) + ")") + params.extend(mid_vals) + tag_by_uid = {} + tag_by_mid = {} + if clauses: + rows_i = _ci.execute( + f"SELECT uid, message_id, tags, spam_verdict FROM email_tags " + f"WHERE folder=? AND {_owner_clause_i} AND {_account_clause_i} AND ({' OR '.join(clauses)})", + params, + ).fetchall() + for uid_i, mid_i, tags_raw_i, spam_i in rows_i: + try: + tags_i = json.loads(tags_raw_i or "[]") + except Exception: + tags_i = [] + if isinstance(tags_i, list): + tags_i = _sanitize_visible_email_tags(tags_i) + entry_i = {"tags": tags_i if isinstance(tags_i, list) else [], "spam": bool(spam_i)} + if uid_i: + tag_by_uid[str(uid_i)] = entry_i + if mid_i: + tag_by_mid[str(mid_i).strip()] = entry_i + _ci.close() + for e in emails: + tag_entry = tag_by_mid.get((e.get("message_id") or "").strip()) or tag_by_uid.get(str(e.get("uid") or "")) + if tag_entry: + e["tags"] = _sanitize_visible_email_tags(tag_entry.get("tags", []), is_answered=bool(e.get("is_answered"))) + e["is_spam_verdict"] = tag_entry.get("spam", False) + except Exception as e: + logger.debug(f"email index tag merge skipped: {e}") + + try: + import sqlite3 as _sql3c + ids = [(e.get("message_id") or "").strip() for e in emails if e.get("message_id")] + if ids: + _ccal = _sql3c.connect(SCHEDULED_DB) + owner_clause, owner_params = _email_cache_owner_clause(owner) + ph = ",".join("?" * len(ids)) + cal_rows = _ccal.execute( + f"SELECT message_id, event_uids FROM email_calendar_extractions " + f"WHERE message_id IN ({ph}) AND {owner_clause}", + (*ids, *owner_params), + ).fetchall() + _ccal.close() + by_mid = {} + for mid, raw_uids in cal_rows: + try: + uids = json.loads(raw_uids or "[]") + except Exception: + uids = [] + if isinstance(uids, list): + by_mid[(mid or "").strip()] = [str(u).strip() for u in uids if str(u).strip()] + for e in emails: + event_uids = by_mid.get((e.get("message_id") or "").strip()) or [] + if event_uids: + e["calendar_event_uids"] = event_uids + except Exception as e: + logger.debug(f"email calendar event link attach skipped: {e}") + + _hide_unlinked_calendar_tags(emails) + if filter_ and filter_.startswith("tag:") and filter_ != "tag:spam": + _final_tag = filter_[len("tag:"):].strip().lower().replace("_", "-") + emails = [e for e in emails if _final_tag in (e.get("tags") or [])] + total = len(emails) + if has_attachments_only: emails = [e for e in emails if e.get("has_attachments")] # Total now reflects matches inside the scanned window, not @@ -1013,17 +1746,25 @@ def setup_email_routes(): except Exception as _summary_err: logger.debug(f"Bulk summary attach skipped: {_summary_err}") - return {"emails": emails, "total": total, "folder": folder, "offset": offset} + return { + "emails": emails, + "total": total, + "folder": folder, + "offset": offset, + "sync": { + "source": "imap", + "indexed": len(cached_rows) if uid_list else 0, + "updated_at": datetime.utcnow().isoformat() + "Z", + }, + } except Exception as e: + conn_ok = False logger.error(f"Failed to list emails: {e}") detail = str(e).strip() return {"emails": [], "total": 0, "error": f"Mail operation failed: {detail[:180]}" if detail else "Mail operation failed"} finally: if conn: - try: - conn.logout() - except Exception: - pass + _pooled_release(account_id, conn, ok=conn_ok, owner=owner) def _related_thread_attachments_sync( folder: str, @@ -1098,9 +1839,13 @@ def setup_email_routes(): ): """List emails. Uses an 8s in-memory cache + offloads blocking IMAP calls to a worker thread so the event loop never stalls.""" + started_at = _time.monotonic() _deferred = getattr(_start_poller, '_deferred', None) if _deferred: await _deferred() + fixture_result = _fixture_email_list(folder, limit, offset, filter, from_addr, owner) + if fixture_result is not None: + return fixture_result # SECURITY: include `owner` in the cache key so two users with # different account scopes don't share a cached list. ck = _list_cache_key(account_id, folder, filter, limit, offset, from_addr or "") + (int(bool(has_attachments)), owner) @@ -1108,6 +1853,16 @@ def setup_email_routes(): cached = _list_cache_get(ck) if cached is not None: _schedule_recent_email_warm(cached.get("emails") or [], folder, account_id, owner) + cached = dict(cached) + sync_meta = dict(cached.get("sync") or {}) + sync_meta["source"] = "memory_cache" + cached["sync"] = sync_meta + elapsed_ms = int((_time.monotonic() - started_at) * 1000) + if elapsed_ms > 500: + logger.info( + "email list cache hit slow owner=%s account=%s folder=%s filter=%s limit=%s offset=%s total=%sms", + owner, account_id or "", folder, filter, limit, offset, elapsed_ms, + ) return cached result = await _asyncio.to_thread( _list_emails_sync, folder, limit, offset, filter, account_id, from_addr, @@ -1118,8 +1873,89 @@ def setup_email_routes(): _record_email_received_events(owner, account_id, folder, result.get("emails") or []) _schedule_recent_email_warm(result.get("emails") or [], folder, account_id, owner) _list_cache_put(ck, result) + elapsed_ms = int((_time.monotonic() - started_at) * 1000) + if elapsed_ms > 1500: + logger.warning( + "Slow email list owner=%s account=%s folder=%s filter=%s limit=%s offset=%s cache_bust=%s emails=%s total=%s elapsed=%sms", + owner, account_id or "", folder, filter, limit, offset, bool(cache_bust), + len((result or {}).get("emails") or []), (result or {}).get("total"), elapsed_ms, + ) return result + @router.get("/unread-state") + async def unread_state( + folder: str = Query("INBOX"), + account_id: str | None = Query(None), + owner: str = Depends(require_owner), + ): + """Cheap unread summary for notification dots. + + Reads the local message index first so periodic UI polling does not + trigger Gmail SEARCH/LIST round-trips. If no local index exists for the + account/folder yet, do one tiny IMAP fallback and then cache naturally + through the list path. + """ + fixture_result = _fixture_email_list(folder, 1, 0, "unread", None, owner) + if fixture_result is not None: + return { + "unread_count": int(fixture_result.get("total") or 0), + "max_uid": max([int(e.get("uid") or 0) for e in _fixture_email_rows(owner)] or [0]), + "folder": folder, + "sync": {"source": "fixture"}, + } + try: + account_key = _account_cache_key(account_id, owner) + conn = _sql3.connect(SCHEDULED_DB) + try: + row = conn.execute( + """ + SELECT COUNT(*), + MAX(CASE WHEN uid GLOB '[0-9]*' THEN CAST(uid AS INTEGER) ELSE 0 END), + MAX(updated_at) + FROM email_message_index + WHERE owner=? AND account_key=? AND folder=? + AND (flags IS NULL OR instr(flags, '\\Seen') = 0) + """, + (owner or "", account_key, folder), + ).fetchone() + total_row = conn.execute( + "SELECT COUNT(*), MAX(updated_at) FROM email_message_index WHERE owner=? AND account_key=? AND folder=?", + (owner or "", account_key, folder), + ).fetchone() + finally: + conn.close() + indexed_total = int((total_row or [0])[0] or 0) + if indexed_total: + return { + "unread_count": int((row or [0])[0] or 0), + "max_uid": int((row or [0, 0])[1] or 0), + "folder": folder, + "sync": { + "source": "index", + "indexed": indexed_total, + "updated_at": (total_row or [None, None])[1], + }, + } + except Exception: + logger.debug("unread-state index lookup skipped", exc_info=True) + + result = await _asyncio.to_thread( + _list_emails_sync, folder, 1, 0, "unread", account_id, None, False, owner, + ) + emails = (result or {}).get("emails") or [] + max_uid = 0 + if emails: + try: + max_uid = max(int(e.get("uid") or 0) for e in emails) + except Exception: + max_uid = 0 + return { + "unread_count": int((result or {}).get("total") or len(emails) or 0), + "max_uid": max_uid, + "folder": folder, + "sync": {"source": "imap_fallback"}, + } + @router.post("/{uid}/unflag-spam") async def unflag_spam(uid: str, owner: str = Depends(require_owner)): """User override — mark email as not spam.""" @@ -1191,6 +2027,7 @@ def setup_email_routes(): folder: str = Query("INBOX"), limit: int = Query(50), account_id: str | None = Query(None), + local_only: bool = Query(False), owner: str = Depends(require_owner), ): """Search emails server-side via IMAP SEARCH. Matches subject, from, or body text. @@ -1204,7 +2041,23 @@ def setup_email_routes(): # CRLF in q would terminate the IMAP command early — reject defensively. if "\r" in q or "\n" in q: raise HTTPException(400, "Invalid query") + indexed_response = None try: + indexed_emails, indexed_total, indexed_at = _email_index_search(owner, account_id, folder, q, limit) + indexed_response = { + "emails": indexed_emails, + "total": indexed_total, + "query": q, + "folder": folder, + "source": "index", + "sync": { + "source": "index", + "updated_at": indexed_at, + }, + } + if local_only: + return indexed_response + with _imap(account_id, owner=owner) as conn: # If the user asked for INBOX, try to upgrade to All Mail — # one folder == every email on Gmail-class servers. @@ -1234,14 +2087,33 @@ def setup_email_routes(): status, data = _imap_uid_search(conn, search_cmd) if status != "OK" or not data[0]: - return {"emails": [], "total": 0, "query": q, "folder": effective_folder} + return { + "emails": [], + "total": 0, + "query": q, + "folder": effective_folder, + "source": "imap", + "sync": { + "source": "imap", + "updated_at": datetime.utcnow().isoformat() + "Z", + }, + } uid_list = data[0].split() total = len(uid_list) uid_list = list(reversed(uid_list))[:limit] + uid_order = [uid.decode(errors="ignore") if isinstance(uid, bytes) else str(uid) for uid in uid_list] + cached_rows = _email_index_rows(owner, account_id, effective_folder, uid_order) emails = [] - for uid in uid_list: + fetched_emails = [] + for uid in uid_order: + cached = cached_rows.get(uid) + if cached: + item = dict(cached) + item["folder"] = effective_folder + emails.append(item) + continue try: status, msg_data = _imap_uid_fetch(conn, uid, "(UID FLAGS RFC822.HEADER)") if status != "OK": @@ -1259,84 +2131,80 @@ def setup_email_routes(): flags = flag_match.group(1).decode(errors="replace") if not raw_header: continue - msg = email_mod.message_from_bytes(raw_header) - subject = _decode_header(msg.get("Subject", "(no subject)")) - sender = _decode_header(msg.get("From", "unknown")) - date_str = msg.get("Date", "") - message_id = msg.get("Message-ID", "") - sender_name, sender_addr = email.utils.parseaddr(sender) - to_str = _decode_header(msg.get("To", "")) - cc_str = _decode_header(msg.get("Cc", "")) - parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None - if parsed_date and parsed_date.tzinfo is None: - from datetime import timezone as _tz - parsed_date = parsed_date.replace(tzinfo=_tz.utc) - iso_date = parsed_date.isoformat() if parsed_date else "" - date_epoch = parsed_date.timestamp() if parsed_date else 0.0 - ct = msg.get("Content-Type", "") - has_attachments = "multipart/mixed" in ct.lower() or "multipart/related" in ct.lower() - - stable_uid = "" - for part in msg_data: - if isinstance(part, tuple): - meta_b = part[0] if isinstance(part[0], bytes) else str(part[0]).encode() - stable_uid = _uid_from_fetch_meta(meta_b) or stable_uid - if not stable_uid: + parsed = None + for meta_b, payload in _group_uid_fetch_records(msg_data): + parsed = _parse_email_list_record(meta_b, payload) + if parsed: + break + if not parsed: continue - emails.append({ - "uid": stable_uid, - "message_id": message_id.strip(), - "subject": subject, - "from_name": sender_name or sender_addr, - "from_address": sender_addr, - "to": to_str, - "cc": cc_str, - "date": iso_date, - "date_display": date_str, - "date_epoch": date_epoch, - "is_read": "\\Seen" in flags, - "is_answered": "\\Answered" in flags, - "is_flagged": "\\Flagged" in flags, - "flags": flags, - "has_attachments": has_attachments, - # Stamp the folder so the frontend opens each - # email from the folder it actually lives in - # (the search may have run against All Mail - # even though the caller asked for INBOX), - # otherwise clicks open whatever happens to - # have the same UID in INBOX → wrong email. - "folder": effective_folder, - }) + parsed["folder"] = effective_folder + emails.append(parsed) + fetched_emails.append(parsed) except Exception as e: logger.warning(f"Error parsing search result {uid}: {e}") continue + _email_index_upsert(owner, account_id, effective_folder, fetched_emails) - return {"emails": emails, "total": total, "query": q} + return { + "emails": emails, + "total": total, + "query": q, + "folder": effective_folder, + "source": "imap", + "sync": { + "source": "imap", + "updated_at": datetime.utcnow().isoformat() + "Z", + }, + } except Exception as e: logger.error(f"Search failed: {e}") + if indexed_response and indexed_response.get("emails"): + indexed_response["fallback"] = True + return indexed_response return {"emails": [], "total": 0, "error": "Mail operation failed"} - def _read_email_sync(uid, folder, account_id, owner, mark_seen=True): + def _read_email_sync(uid, folder, account_id, owner, mark_seen=True, full=False): """Sync IMAP read — wrapped in to_thread by the async handler. - Two-phase: read body in readonly to avoid races with concurrent reads - of the same UID, then flip \\Seen in a separate readwrite session. - BODY.PEEK[] keeps the fetch itself from tripping \\Seen. + The normal reader path fetches the headers plus a bounded body prefix. + That avoids downloading multi-megabyte attachments just to open a + message. Full-message fetch remains available for flows that need + attachment metadata immediately, such as forwarding. """ import time as _t _t0 = _t.monotonic() raw = None + preview_bytes = 384 * 1024 _t_select = 0.0 _t_fetch = 0.0 try: with _imap(account_id, owner=owner) as conn: conn.select(_q(folder), readonly=True) _t_select = _t.monotonic() - _t0 - status, msg_data = _imap_uid_fetch(conn, uid, "(BODY.PEEK[])") + fetch_query = "(BODY.PEEK[])" if full else f"(BODY.PEEK[HEADER] BODY.PEEK[TEXT]<0.{preview_bytes}>)" + status, msg_data = _imap_uid_fetch(conn, uid, fetch_query) _t_fetch = _t.monotonic() - _t0 if status != "OK": return {"error": f"Email UID {uid} not found"} - raw = msg_data[0][1] + if full: + raw = msg_data[0][1] + else: + header_part = b"" + text_part = b"" + for item in msg_data or []: + if not isinstance(item, tuple) or len(item) < 2: + continue + meta_b = item[0] if isinstance(item[0], bytes) else str(item[0]).encode() + payload = item[1] or b"" + meta_up = meta_b.upper() + if b"BODY[HEADER]" in meta_up: + header_part = payload + elif b"BODY[TEXT]" in meta_up: + text_part = payload + if not header_part and msg_data and isinstance(msg_data[0], tuple): + header_part = msg_data[0][1] or b"" + raw = header_part + b"\r\n" + text_part msg = email_mod.message_from_bytes(raw) @@ -1353,9 +2221,9 @@ def setup_email_routes(): sender_name, sender_addr = email.utils.parseaddr(sender) parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None - attachments = _list_attachments_from_msg(msg) + attachments = _list_attachments_from_msg(msg) if full else (_email_attachment_meta_cache_get(owner, account_id, folder, uid) or []) related_attachments = [] - if not _has_visible_attachments(msg): + if full and not _has_visible_attachments(msg): related_attachments = _related_thread_attachments_sync( folder, account_id, @@ -1365,22 +2233,15 @@ def setup_email_routes(): in_reply_to, references, ) + if attachments: + _email_attachment_meta_cache_put(owner, account_id, folder, uid, message_id, attachments) - if mark_seen: - # Set \Seen in a separate readwrite session so concurrent reads - # of the same UID don't fight over a shared SELECT state. - try: - with _imap(account_id, owner=owner) as conn2: - conn2.select(_q(folder)) - conn2.uid("STORE", _uid_bytes(uid), "+FLAGS", "\\Seen") - except Exception: - pass _t_total = _t.monotonic() - _t0 if _t_total > 2.0: logger.warning( f"Slow email read uid={uid} folder={folder} " f"select={_t_select*1000:.0f}ms fetch={_t_fetch*1000:.0f}ms " - f"size={len(raw)} total={_t_total*1000:.0f}ms" + f"size={len(raw)} full={full} total={_t_total*1000:.0f}ms" ) # Look up cached summary, AI reply, and LLM-detected boundaries @@ -1473,6 +2334,7 @@ def setup_email_routes(): "body": body, "body_html": body_html, "attachments": attachments, + "attachments_deferred": not full and not attachments, "related_attachments": related_attachments, "attachment_version": EMAIL_READ_ATTACHMENT_VERSION, "cached_summary": cached_summary, @@ -1490,7 +2352,8 @@ def setup_email_routes(): with _imap(account_id, owner=owner) as conn: conn.select(_q(folder)) conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "\\Seen") - _invalidate_list_cache(account_id, folder) + _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) + _update_list_cache_seen(account_id, folder, uid, True) except Exception as e: logger.debug(f"mark-seen after cached read failed uid={uid}: {e}") @@ -1500,10 +2363,14 @@ def setup_email_routes(): folder: str = Query("INBOX"), account_id: str | None = Query(None), mark_seen: bool = Query(True), + full: bool = Query(False), owner: str = Depends(require_owner), ): """Read email body. Cached for 30m, sync IMAP work runs in a thread.""" - ck = _read_cache_key(account_id, folder, uid, owner=owner) + fixture_result = _fixture_email_read(uid, folder, owner) + if fixture_result is not None: + return fixture_result + ck = _read_cache_key(account_id, folder, uid, owner=owner) + (int(bool(full)),) cached = _read_cache_get(ck) if cached is not None: # Older cached read responses lack the thread-attachment fallback. @@ -1518,9 +2385,26 @@ def setup_email_routes(): except RuntimeError: pass return cached - result = await _asyncio.to_thread(_read_email_sync, uid, folder, account_id, owner, mark_seen) + if not full: + persisted = _email_preview_cache_get(owner, account_id, folder, uid) + if persisted and persisted.get("attachment_version") == EMAIL_READ_ATTACHMENT_VERSION: + _read_cache_put(ck, persisted) + if mark_seen: + try: + _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner)) + except RuntimeError: + pass + return persisted + result = await _asyncio.to_thread(_read_email_sync, uid, folder, account_id, owner, mark_seen, full) if result and not result.get("error"): _read_cache_put(ck, result) + if not full: + _email_preview_cache_put(owner, account_id, folder, uid, result) + if mark_seen: + try: + _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner)) + except RuntimeError: + pass return result def _schedule_recent_email_warm(emails: list, folder: str, account_id: str | None, owner: str): @@ -1544,7 +2428,7 @@ def setup_email_routes(): size = 0 if size > _WARM_MAX_BYTES: continue - ck = _read_cache_key(account_id, folder, uid, owner=owner) + ck = _read_cache_key(account_id, folder, uid, owner=owner) + (0,) if _read_cache_get(ck) is not None or ck in _WARMING_READS: continue _WARMING_READS.add(ck) @@ -1555,19 +2439,20 @@ def setup_email_routes(): return async def _warm(): + await _asyncio.sleep(3.0) for uid, ck in selected: if _read_cache_get(ck) is not None: _WARMING_READS.discard(ck) continue try: - result = await _asyncio.to_thread(_read_email_sync, uid, folder, account_id, owner, False) + result = await _asyncio.to_thread(_read_email_sync, uid, folder, account_id, owner, False, False) if result and not result.get("error"): _read_cache_put(ck, result) except Exception as e: logger.debug(f"email read warm skipped uid={uid}: {e}") finally: _WARMING_READS.discard(ck) - await _asyncio.sleep(0.05) + await _asyncio.sleep(0.35) try: _asyncio.create_task(_warm()) @@ -1577,6 +2462,9 @@ def setup_email_routes(): @router.get("/attachments/{uid}") async def list_attachments(uid: str, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)): """List attachments for an email.""" + cached = _email_attachment_meta_cache_get(owner, account_id, folder, uid) + if cached is not None: + return {"attachments": cached, "uid": uid, "sync": {"source": "attachment_metadata_cache"}} try: with _imap(account_id, owner=owner) as conn: conn.select(_q(folder), readonly=True) @@ -1586,6 +2474,7 @@ def setup_email_routes(): raw = msg_data[0][1] msg = email_mod.message_from_bytes(raw) attachments = _list_attachments_from_msg(msg) + _email_attachment_meta_cache_put(owner, account_id, folder, uid, msg.get("Message-ID", ""), attachments) return {"attachments": attachments, "uid": uid} except Exception as e: logger.error(f"Failed to list attachments for {uid}: {e}") @@ -1618,6 +2507,57 @@ def setup_email_routes(): logger.error(f"Failed to download attachment {uid}/{index}: {e}") return {"error": "Mail operation failed"} + @router.get("/inline-image/{uid}") + async def inline_image( + uid: str, + cid: str = Query(...), + folder: str = Query("INBOX"), + account_id: str | None = Query(None), + owner: str = Depends(require_owner), + ): + """Serve an inline MIME image by Content-ID after the user explicitly clicks Load.""" + want = (cid or "").strip().strip("<>") + if not want: + raise HTTPException(status_code=400, detail="Missing image Content-ID") + try: + with _imap(account_id, owner=owner) as conn: + conn.select(_q(folder), readonly=True) + status, msg_data = _imap_uid_fetch(conn, uid, "(RFC822)") + if status != "OK": + raise HTTPException(status_code=404, detail="Email not found") + raw = msg_data[0][1] + msg = email_mod.message_from_bytes(raw) + idx = 0 + for part in msg.walk(): + cd = str(part.get("Content-Disposition", "")) + ct = part.get_content_type() + is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename()) + if part.is_multipart() and not is_attached_email: + continue + if ct in ("text/plain", "text/html") and "attachment" not in cd: + continue + content_id = (part.get("Content-ID") or "").strip().strip("<>") + if content_id != want: + idx += 1 + continue + if not ct.lower().startswith("image/"): + raise HTTPException(status_code=415, detail="Content-ID is not an image") + target_dir = attachment_extract_dir(folder, uid) + filepath = _extract_attachment_to_disk(msg, idx, target_dir) + if not filepath: + raise HTTPException(status_code=404, detail="Inline image not found") + return FileResponse( + path=str(filepath), + media_type=ct, + headers={"Content-Disposition": f'inline; filename="{filepath.name}"'}, + ) + raise HTTPException(status_code=404, detail="Inline image not found") + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to load inline image {uid}/{want}: {e}") + return {"error": "Mail operation failed"} + @router.post("/attachment-as-doc/{uid}/{index}") async def attachment_as_doc(uid: str, index: int, request: Request, folder: str = Query("INBOX"), account_id: str | None = Query(None), owner: str = Depends(require_owner)): """Extract an email attachment and open it in the document editor. @@ -1927,6 +2867,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Seen", add=False): return {"success": False, "error": "Email not found"} + _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", False) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -1943,6 +2884,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)): return {"success": False, "error": "Email not found"} + _email_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on)) _invalidate_list_cache(account_id, folder) return {"success": True, "flagged": bool(on)} except Exception as e: @@ -1957,6 +2899,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Seen", add=True): return {"success": False, "error": "Email not found"} + _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -1973,6 +2916,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, "Archive", role="archive"): return {"success": False, "error": "Email not found"} + _email_index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -1987,6 +2931,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, "Trash", role="trash"): return {"success": False, "error": "Email not found"} + _email_index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -2002,6 +2947,7 @@ def setup_email_routes(): if not _store_email_flag(conn, uid, "\\Deleted", add=True): return {"success": False, "error": "Email not found"} conn.expunge() + _email_index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -2091,6 +3037,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, dest): return {"success": False, "error": f"Failed to move to {dest}"} + _email_index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -2100,6 +3047,15 @@ def setup_email_routes(): @router.get("/folders") async def list_folders(account_id: str | None = Query(None), owner: str = Depends(require_owner)): """List IMAP folders.""" + if _fixture_email_enabled(): + return {"folders": ["INBOX", "Archive", "Sent"], "sync": {"source": "fixture"}} + cached = _folder_cache_get(account_id, owner) + if cached is not None: + payload = dict(cached) + sync_meta = dict(payload.get("sync") or {}) + sync_meta["source"] = "folder_cache" + payload["sync"] = sync_meta + return payload try: with _imap(account_id, owner=owner) as conn: status, folders = conn.list() @@ -2110,7 +3066,9 @@ def setup_email_routes(): if match: name = match.group(1) or match.group(2) result.append(name) - return {"folders": result} + payload = {"folders": result, "sync": {"source": "imap", "updated_at": datetime.utcnow().isoformat() + "Z"}} + _folder_cache_put(account_id, owner, payload) + return payload except Exception as e: logger.error(f"list_folders failed: {e}") return {"folders": [], "error": "Mail operation failed"} @@ -2123,6 +3081,9 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Answered", add=True): return {"success": False, "error": "Email not found"} + _email_index_update_flags(owner, account_id, folder, uid, "\\Answered", True) + _clear_done_response_tags(owner, account_id, folder, uid) + _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: logger.error(f"Failed to mark answered {uid}: {e}") @@ -2136,6 +3097,8 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Answered", add=False): return {"success": False, "error": "Email not found"} + _email_index_update_flags(owner, account_id, folder, uid, "\\Answered", False) + _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: logger.error(f"Failed to clear answered {uid}: {e}") @@ -2164,6 +3127,46 @@ def setup_email_routes(): logger.error(f"Failed to upload attachment: {e}") return {"success": False, "error": "Mail operation failed"} + @router.post("/compose-from-attachment/{uid}/{index}") + async def compose_from_attachment( + uid: str, + index: int, + folder: str = Query("INBOX"), + account_id: str | None = Query(None), + owner: str = Depends(require_owner), + ): + """Stage an existing email attachment as a compose upload. + + Used by Forward so original attachments are sent as real attachments, + not just shown as source-email chips. + """ + try: + with _imap(account_id, owner=owner) as conn: + conn.select(_q(folder), readonly=True) + status, msg_data = _imap_uid_fetch(conn, uid, "(RFC822)") + if status != "OK": + return {"success": False, "error": "Email not found"} + raw = msg_data[0][1] + msg = email_mod.message_from_bytes(raw) + target_dir = attachment_extract_dir(folder, uid) + filepath = _extract_attachment_to_disk(msg, index, target_dir) + if not filepath: + return {"success": False, "error": f"Attachment index {index} not found"} + safe_name = re.sub(r"[^\w\s\-.]", "_", filepath.name or "attachment").strip() or "attachment" + token = f"{uuid.uuid4().hex}_{safe_name}" + dest = COMPOSE_UPLOADS_DIR / token + import shutil as _shutil + _shutil.copyfile(str(filepath), str(dest)) + return { + "success": True, + "token": token, + "filename": safe_name, + "size": dest.stat().st_size, + } + except Exception as e: + logger.error(f"Failed to stage forwarded attachment {uid}/{index}: {e}") + return {"success": False, "error": "Mail operation failed"} + @router.delete("/compose-upload/{token}") async def delete_compose_upload(token: str, owner: str = Depends(require_owner)): """Delete a staged compose upload.""" @@ -2530,6 +3533,8 @@ def setup_email_routes(): _account_id = cfg.get("account_id") or req.account_id # capture for the IMAP append in the closure _in_reply_to = (req.in_reply_to or "").strip() + _source_uid = (req.source_uid or "").strip() + _source_folder = (req.source_folder or "INBOX").strip() or "INBOX" _oauth_provider = cfg.get("oauth_provider") or "" _oauth_access_token = cfg.get("oauth_access_token") or "" _oauth_refresh_token = cfg.get("oauth_refresh_token") or "" @@ -2583,6 +3588,18 @@ def setup_email_routes(): pass # Auto-mark the source email as Answered/done so it # disappears from "undone" filters. + if _source_uid: + try: + st, _sel = imap.select(_q(_source_folder), readonly=False) + if st == "OK" and _store_email_flag(imap, _source_uid, "\\Answered", add=True): + _email_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True) + _clear_done_response_tags(owner, _account_id, _source_folder, _source_uid) + _invalidate_list_cache(_account_id, _source_folder) + logger.info(f"Marked source UID {_source_uid} as \\Answered in {_source_folder}") + else: + logger.warning(f"Failed to mark source UID {_source_uid} as answered in {_source_folder}") + except Exception as e: + logger.warning(f"Failed to mark exact source UID as answered: {e}") if _in_reply_to: try: # Strip any angle brackets and quote for IMAP @@ -2914,6 +3931,96 @@ def setup_email_routes(): logger.error(f"Failed to summarize: {e}") return {"success": False, "error": "Mail operation failed"} + @router.post("/translate") + async def translate_email(data: dict, owner: str = Depends(require_owner)): + """Translate an email body into a target language.""" + try: + from src.endpoint_resolver import ( + resolve_endpoint, + resolve_utility_fallback_candidates, + resolve_chat_fallback_candidates, + ) + from src.llm_core import llm_call_async_with_fallback + + body = (data.get("body") or "").strip() + subject = (data.get("subject") or "").strip() + sender = (data.get("from") or "").strip() + target_language = (data.get("target_language") or "English").strip() or "English" + auto = bool(data.get("auto", False)) + if not body: + return {"success": False, "error": "No body provided"} + + candidates = [] + seen = set() + + def _add(url, model, headers): + key = (url or "", model or "") + if not url or not model or key in seen: + return + seen.add(key) + candidates.append((url, model, headers)) + + try: + _add(*resolve_endpoint("utility", owner=owner)) + except Exception: + pass + try: + _add(*resolve_endpoint("default", owner=owner)) + except Exception: + pass + for cand in resolve_utility_fallback_candidates(owner=owner) or []: + _add(*cand) + for cand in resolve_chat_fallback_candidates(owner=owner) or []: + _add(*cand) + if not candidates: + return {"success": False, "error": "No LLM endpoint configured"} + + content = await llm_call_async_with_fallback( + candidates, + messages=[ + { + "role": "system", + "content": ( + "You translate emails faithfully. Preserve meaning, names, dates, money, addresses, " + "bullet structure, and tone. Do not summarize or answer the email. " + "Output only the translation between <<>> and <<>>. " + "If AUTO mode is enabled and the email is already primarily in the target language, " + "output exactly <<>>." + ), + }, + { + "role": "user", + "content": ( + f"AUTO mode: {'enabled' if auto else 'disabled'}\n" + f"Target language: {target_language}\n\n" + f"From: {sender}\nSubject: {subject}\n\n{body[:16000]}\n\n" + "Translate the email unless AUTO mode is enabled and it is already primarily in the target language.\n" + "Return only:\n<<>>\ntranslated text\n<<>>" + ), + }, + ], + temperature=0.2, + max_tokens=8192, + timeout=180, + ) + model = candidates[0][1] if candidates else "" + content = (content or "").strip() + content = _extract_reply(content) + if "<<>>" in content: + return {"success": True, "same_language": True, "language": target_language, "model_used": model} + marker = re.search(r"<<>>\s*(.*?)\s*<<>>", content, re.S | re.I) + if marker: + content = marker.group(1).strip() + else: + content = re.sub(r"^\s*<<>>\s*", "", content, flags=re.I).strip() + content = re.sub(r"\s*<<>>\s*$", "", content, flags=re.I).strip() + if not content: + return {"success": False, "error": "Empty response from model"} + return {"success": True, "translation": content, "language": target_language, "model_used": model} + except Exception as e: + logger.error(f"Failed to translate email: {e}") + return {"success": False, "error": "Mail operation failed"} + @router.post("/ai-reply") async def ai_reply(data: dict, owner: str = Depends(require_owner)): """Generate an AI-drafted reply to an email using the user's writing style.""" @@ -3190,6 +4297,8 @@ def setup_email_routes(): cfg["email_auto_tag"] = bool(settings.get("email_auto_tag", False)) cfg["email_auto_spam"] = bool(settings.get("email_auto_spam", False)) cfg["email_auto_calendar"] = bool(settings.get("email_auto_calendar", False)) + cfg["email_auto_translate"] = bool(settings.get("email_auto_translate", False)) + cfg["email_translate_language"] = settings.get("email_translate_language", "English") return cfg @router.put("/config") @@ -3203,9 +4312,11 @@ def setup_email_routes(): """ # Automation flags stay in settings.json (they're global, not per-account) settings = _load_settings() - for key in ["email_auto_summarize", "email_auto_reply", "email_auto_tag", "email_auto_spam", "email_auto_calendar"]: + for key in ["email_auto_summarize", "email_auto_reply", "email_auto_tag", "email_auto_spam", "email_auto_calendar", "email_auto_translate"]: if key in data: settings[key] = data[key] + if "email_translate_language" in data: + settings["email_translate_language"] = (data.get("email_translate_language") or "English").strip() or "English" _save_settings(settings) # Credentials go into the default account row diff --git a/routes/history_routes.py b/routes/history_routes.py index 59ed6674e..2324ae286 100644 --- a/routes/history_routes.py +++ b/routes/history_routes.py @@ -3,7 +3,8 @@ import json import uuid import logging -from typing import Dict, Any +import re +from typing import Dict, Any, Optional from fastapi import APIRouter, Request, HTTPException @@ -19,6 +20,63 @@ from routes.session_routes import ( logger = logging.getLogger(__name__) +_HISTORY_INLINE_MEDIA_THRESHOLD = 200_000 +_DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+") + + +def _history_display_content(content: Any) -> Any: + """Return a lightweight browser-display copy of stored message content. + + Older multimodal user messages may be persisted as a JSON *string* + containing image_url blocks with inline base64 image bytes. Those bytes are + needed for model calls when the turn is first sent, but they should not be + sent back through /api/history every time the user opens the chat. The + attachment metadata already carries file ids/names for the UI cards. + """ + if isinstance(content, list): + text_parts = [] + omitted_media = 0 + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "text": + text = block.get("text") + if isinstance(text, str) and text: + text_parts.append(text) + elif block.get("type") in {"image_url", "input_image", "audio", "input_audio"}: + omitted_media += 1 + text = "\n".join(text_parts).strip() + if omitted_media and not text: + return f"[{omitted_media} media attachment{'s' if omitted_media != 1 else ''} omitted from history view]" + return text + + if not isinstance(content, str): + return content + if len(content) < _HISTORY_INLINE_MEDIA_THRESHOLD and "data:image/" not in content: + return content + + stripped = content.lstrip() + if stripped.startswith("["): + try: + blocks = json.loads(content) + except (json.JSONDecodeError, TypeError, ValueError): + blocks = None + if isinstance(blocks, list): + text_parts = [] + for block in blocks: + if not isinstance(block, dict): + continue + if block.get("type") == "text": + text = block.get("text") + if isinstance(text, str) and text: + text_parts.append(text) + if text_parts: + return "\n".join(text_parts).strip() + + if "data:image/" in content: + return _DATA_IMAGE_RE.sub("[inline image omitted from history view]", content) + return content + def _merge_continue_rows_to_delete(db_messages, db1, db2): """DB rows to delete when merging the last two assistant messages. @@ -43,9 +101,69 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2): def setup_history_routes(session_manager) -> APIRouter: router = APIRouter(tags=["history"]) + def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]: + entry = {"role": m.role, "content": _history_display_content(m.content)} + meta = {} + if m.meta_data: + try: + meta = json.loads(m.meta_data) or {} + except (json.JSONDecodeError, ValueError): + meta = {} + if m.timestamp and "timestamp" not in meta: + meta["timestamp"] = m.timestamp.isoformat() + "Z" + if meta: + entry["metadata"] = meta + return entry + @router.get("/api/history/{session_id}") - async def get_session_history(request: Request, session_id: str) -> Dict[str, Any]: + async def get_session_history( + request: Request, + session_id: str, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Dict[str, Any]: _verify_session_owner(request, session_id) + if limit is not None: + page_limit = max(1, min(int(limit), 100)) + db = SessionLocal() + try: + db_session = db.query(DbSession).filter(DbSession.id == session_id).first() + if db_session is None: + raise HTTPException(404, f"Session '{session_id}' not found") + + total = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .count() + ) + page_offset = int(offset) if offset is not None else max(total - page_limit, 0) + page_offset = max(0, min(page_offset, total)) + rows = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .order_by(DbChatMessage.timestamp) + .offset(page_offset) + .limit(page_limit) + .all() + ) + history_dict = [ + entry for entry in (_db_history_entry(m) for m in rows) + if not (entry.get("metadata") or {}).get("hidden") + ] + return { + "history": history_dict, + "model": db_session.model, + "endpoint_url": db_session.endpoint_url, + "name": db_session.name, + "offset": page_offset, + "limit": page_limit, + "total": total, + "has_more_before": page_offset > 0, + "has_more_after": page_offset + len(rows) < total, + } + finally: + db.close() + try: session = session_manager.get_session(session_id) except KeyError: @@ -57,7 +175,7 @@ def setup_history_routes(session_manager) -> APIRouter: # Skip hidden messages (e.g. compaction summaries for AI context) if msg.metadata and msg.metadata.get("hidden"): continue - entry = {"role": msg.role, "content": msg.content} + entry = {"role": msg.role, "content": _history_display_content(msg.content)} if msg.metadata: entry["metadata"] = msg.metadata history_dict.append(entry) @@ -66,7 +184,7 @@ def setup_history_routes(session_manager) -> APIRouter: continue entry = { "role": msg.get("role", ""), - "content": msg.get("content", ""), + "content": _history_display_content(msg.get("content", "")), } if msg.get("metadata"): entry["metadata"] = msg["metadata"] @@ -82,21 +200,9 @@ def setup_history_routes(session_manager) -> APIRouter: .order_by(DbChatMessage.timestamp) .all() ) - import json as _json db_history = [] for m in db_messages: - entry = {"role": m.role, "content": m.content} - meta = {} - if m.meta_data: - try: - meta = _json.loads(m.meta_data) or {} - except (json.JSONDecodeError, ValueError): - meta = {} - if m.timestamp and "timestamp" not in meta: - meta["timestamp"] = m.timestamp.isoformat() + "Z" - if meta: - entry["metadata"] = meta - db_history.append(entry) + db_history.append(_db_history_entry(m)) if db_history: # Rebuild in-memory history from the full set so hidden # messages (e.g. compaction summaries) are kept for AI context. diff --git a/routes/model_routes.py b/routes/model_routes.py index fb9555438..136b2e9d7 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -18,6 +18,7 @@ from pydantic import BaseModel from fastapi.responses import StreamingResponse from core.database import SessionLocal, ModelEndpoint, Session as DbSession from core.middleware import require_admin +from src.constants import COOKBOOK_STATE_FILE from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS from src.tls_overrides import llm_verify from src.settings import load_settings as _load_settings, save_settings as _save_settings @@ -111,6 +112,67 @@ def _clear_endpoint_settings_for_endpoint(settings: dict, ep_id: str, *, include return cleared +_COOKBOOK_ACTIVE_SERVE_STATUSES = { + "starting", "loading", "ready", "running", "restarting", +} + + +def _active_cookbook_endpoint_ids() -> set[str]: + """Endpoint IDs owned by active Cookbook serve tasks. + + Cookbook auto-registers endpoints with ids like ``local-*``. Those rows are + managed lifecycle state, not durable user configuration. If a tmux stream is + stopped or an old task lingers, the row must stop participating in model + selection and defaults. + """ + try: + if not os.path.exists(COOKBOOK_STATE_FILE): + return set() + with open(COOKBOOK_STATE_FILE, "r", encoding="utf-8") as fh: + raw = fh.read() + state = json.loads(raw) + except Exception: + return set() + out: set[str] = set() + for task in state.get("tasks") or []: + if not isinstance(task, dict) or task.get("type") != "serve": + continue + if str(task.get("status") or "").lower() not in _COOKBOOK_ACTIVE_SERVE_STATUSES: + continue + ep_id = task.get("_endpointId") or task.get("endpointId") or task.get("endpoint_id") + if ep_id: + out.add(str(ep_id)) + return out + + +def _disable_stale_cookbook_local_endpoints(db) -> int: + """Disable enabled cookbook endpoints whose serve task is no longer active.""" + active_ids = _active_cookbook_endpoint_ids() + if not active_ids: + return 0 + stale = ( + db.query(ModelEndpoint) + .filter(ModelEndpoint.is_enabled == True) # noqa: E712 + .filter(ModelEndpoint.id.like("local-%")) + .filter(~ModelEndpoint.id.in_(active_ids)) + .all() + ) + if not stale: + return 0 + settings = _load_settings() + touched_settings = False + for ep in stale: + ep.is_enabled = False + ep.model_refresh_mode = "disabled" + if _clear_endpoint_settings_for_endpoint(settings, ep.id): + touched_settings = True + logger.info("Disabled stale Cookbook endpoint %s (%s @ %s)", ep.id, ep.name, ep.base_url) + if touched_settings: + _save_settings(settings) + db.commit() + return len(stale) + + def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int: """Remove endpoint references from scoped or legacy-flat user preferences.""" if not isinstance(all_prefs, dict): @@ -124,7 +186,24 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int: return cleared_users -def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint_ids) -> bool: +def _endpoint_visible_model_ids(ep: Any) -> List[str]: + """Known visible model ids for an endpoint, including pinned/manual ids.""" + if ep is None: + return [] + return _visible_models( + getattr(ep, "cached_models", None), + getattr(ep, "hidden_models", None), + getattr(ep, "pinned_models", None), + ) + + +def _default_endpoint_needs_assignment( + current_default_id: str, + enabled_endpoint_ids, + *, + current_default_endpoint: Any = None, + current_default_model: str = "", +) -> bool: """Whether the global default chat endpoint should be (re)assigned. True when nothing is configured yet, or the configured default no longer @@ -136,7 +215,12 @@ def _default_endpoint_needs_assignment(current_default_id: str, enabled_endpoint """ if not current_default_id: return True - return current_default_id not in enabled_endpoint_ids + if current_default_id not in enabled_endpoint_ids: + return True + if not (current_default_model or "").strip(): + return True + visible = _endpoint_visible_model_ids(current_default_endpoint) + return bool(visible and current_default_model not in visible) # Loopback hosts a user might type for a local model server (LM Studio, @@ -1166,6 +1250,8 @@ def setup_model_routes(model_discovery): db = SessionLocal() changed = False try: + if _disable_stale_cookbook_local_endpoints(db): + changed = True endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all() now = _time.time() groups: Dict[str, Dict[str, Any]] = {} @@ -1239,6 +1325,8 @@ def setup_model_routes(model_discovery): db = SessionLocal() try: + if _disable_stale_cookbook_local_endpoints(db): + _invalidate_models_cache() q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) if owner and not is_admin: # Regular users see: their own endpoints + null-owner @@ -1376,6 +1464,8 @@ def setup_model_routes(model_discovery): db = SessionLocal() try: + if _disable_stale_cookbook_local_endpoints(db): + _invalidate_models_cache() endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all() local_eps = [] for ep in endpoints: @@ -1608,6 +1698,8 @@ def setup_model_routes(model_discovery): require_admin(request) db = SessionLocal() try: + if _disable_stale_cookbook_local_endpoints(db): + _invalidate_models_cache() rows = db.query(ModelEndpoint).order_by(ModelEndpoint.created_at).all() results = [] for r in rows: @@ -1892,7 +1984,18 @@ def setup_model_routes(model_discovery): ModelEndpoint.is_enabled == True # noqa: E712 ).all() } - if _default_endpoint_needs_assignment(settings.get("default_endpoint_id") or "", enabled_ids): + current_default_id = settings.get("default_endpoint_id") or "" + current_default_ep = None + if current_default_id: + current_default_ep = db.query(ModelEndpoint).filter( + ModelEndpoint.id == current_default_id + ).first() + if _default_endpoint_needs_assignment( + current_default_id, + enabled_ids, + current_default_endpoint=current_default_ep, + current_default_model=settings.get("default_model") or "", + ): from src.endpoint_resolver import _first_chat_model settings["default_endpoint_id"] = ep.id settings["default_model"] = _first_chat_model(model_ids) or "" diff --git a/src/agent_loop.py b/src/agent_loop.py index c0b7a38bc..c5311b86a 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -130,7 +130,8 @@ _API_AGENT_RULES = """\ - "Disable/turn off/enable/turn on " (shell, search, research, browser, documents, incognito, etc.) → call `ui_control` with `toggle `. Aliases accepted: shell→bash, search→web, deepresearch→research, documents→document_editor. NEVER record this as a memory — the user wants the toggle flipped, not a note about preferring it. - "Research X" / "do research on X" / "look into Y" / "deep dive on Z" → call `trigger_research` with `topic`. This starts a live job that appears in the Deep Research sidebar (streams progress + final report). **Do NOT use `web_search` for these** — saw the agent do a plain web_search for "do research on X" when the user wanted the deep-research job. "research X" is a deep-research request, not a quick lookup. (web_search is only for a single quick fact mid-task.) Do NOT POST /api/research/start via app_api either — blocked. After starting, tell the user it's running in the Deep Research sidebar. Only if the user explicitly wants it inline/quick should you fall back to web_search. - "Open/show " (documents, library, gallery, email, inbox, sessions, brain/memories, skills, settings, notes, cookbook) → call `ui_control` with `open_panel `. Panel aliases: library/doc/docs/document→documents, images→gallery, mail/inbox/emails→email, chats/history→sessions, memory/memories→brain, preferences→settings, models/serve/serving→cookbook. CRITICAL: "open memory/memories/brain" / "open skills" / "open notes" / "open documents" / "open cookbook" means OPEN THE PANEL — call `ui_control`, NOT a manage/list tool. The "manage_*" tools list contents in chat; `ui_control open_panel` opens the visual modal the user is asking for. -- "Open/start a reply", "open a reply to ", "draft a reply window" for email → find/read the email if needed, then call `ui_control` with `open_email_reply reply`. This opens the same email document compose window as clicking Reply in the Email UI. Do NOT call `reply_to_email` unless the user explicitly gave body text and wants to SEND immediately. +- "Write/draft a reply saying X" for an open/read email → call `ui_control` with `action="open_email_reply"`, the email `uid`/`folder`, `mode="reply"`, and `body` containing the drafted reply. This opens the same email compose document as clicking Reply and DOES NOT send. Do NOT call `reply_to_email` unless the user explicitly says to send immediately. +- "Open/start a reply", "open a reply to ", "draft a reply window" with no requested body → find/read the email if needed, then call `ui_control` with `open_email_reply reply`. - Bulk email actions ("delete all those", "archive these", "mark all read") require a real email tool call. Use `bulk_email` once with UIDs from the latest `list_emails` result and the same `account`; never claim success without the tool result. - Email UIDs are the values after `UID:` in tool output, not list row numbers. For example, row `1.` with `UID: 90186` must use `"90186"`, never `"1"`. - "Last/latest/newest email" means call `list_emails` with `max_results: 1`, `unread_only: false`, and the right `account`, then read the UID returned by that tool if full content is needed. NEVER use a table row number like "#18" as an email UID. @@ -230,7 +231,7 @@ _DOMAIN_RULES = { - For latest/newest email, list with `max_results: 1`, `unread_only: false`, then read the returned UID if needed. - For named mailboxes/accounts, call `list_email_accounts` if needed and pass the exact `account` value. - Bulk email actions use `bulk_email` once with explicit UIDs; do not loop one message at a time. -- "Open/start a reply" means open a draft via `ui_control open_email_reply`; only `reply_to_email` when the user clearly wants to send now.""", +- "Write/draft a reply saying X" means open a pre-filled draft via `ui_control open_email_reply ... ` / structured `body`; only `reply_to_email` when the user clearly wants to send now.""", "cookbook": """\ ## Cookbook/model-serving rules - Cookbook is the LLM-serving subsystem. @@ -446,7 +447,7 @@ List recent emails from a folder, newest first, including read messages by defau ```reply_to_email {"uid": "1234", "body": "Sounds good — talk Friday.", "account": "gmail"} ``` -SEND a reply email immediately by UID. Do not use this for "open a reply" or "start a reply" — those should use `ui_control` with `open_email_reply reply` to open the email draft document. For follow-up requests like "reply ..." after reading/listing email where the user clearly wants to send now, use the exact UID and account from the latest `read_email`/`list_emails` result. Never invent UID `1`. Threads automatically (In-Reply-To/References handled). +SEND a reply email immediately by UID. Do not use this for "write/draft a reply", "open a reply", or "start a reply" — those should use `ui_control` with `open_email_reply reply ` (or structured `body`) to open the email draft document. Only use this when the user explicitly says to send now. Never invent UID `1`. Threads automatically (In-Reply-To/References handled). CRITICAL — signatures: DO NOT invent a sign-off name. End the body with just `Thanks,` or similar — never type a person's name unless the user explicitly told you what to sign as. When `agent_email_confirm` is on (default), the tool returns `{pending: true, pending_id: ...}` and stages the email for the user to approve in the chat UI instead of SMTPing immediately.""", "bulk_email": """\ @@ -477,7 +478,7 @@ If the user asks for a reminder/alarm before the event, pass `reminder_minutes` "send_to_session": "- ```send_to_session``` — Send a message to another session. Line 1 = session_id, rest = message. Use for orchestrating work across sessions.", "search_chats": "- ```search_chats``` — Search past session transcripts for direct conversation evidence. Use when user asks 'did we discuss X?', 'find the conversation about Y', or when prior chat context is more appropriate than persistent memory.", "pipeline": "- ```pipeline``` — Run a multi-step AI pipeline. Args (JSON) with ordered steps, each specifying a model and prompt. Use for complex workflows.", - "ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel ` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply ` (opens an email compose document, does NOT send), `set_mode agent/chat`, `switch_model `, `set_theme `, `create_theme ` (optional key=val for advanced colors AND background effects: bgPattern=, bgEffectColor=#RRGGBB, bgEffectIntensity=, bgEffectSize=, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel `. Built-in theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute. For any other vibe/name, use create_theme.", + "ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel ` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply ` (opens an email compose document pre-filled with body, DOES NOT send; use this for normal “write/draft a reply saying X” requests), `set_mode agent/chat`, `switch_model `, `set_theme `, `create_theme ` (optional key=val for advanced colors AND background effects: bgPattern=, bgEffectColor=#RRGGBB, bgEffectIntensity=, bgEffectSize=, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel `. Built-in theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute. For any other vibe/name, use create_theme.", "ask_user": "- ```ask_user``` — Ask the user a multiple-choice question when the task is genuinely ambiguous and the answer changes what you do next (pick an approach, confirm an assumption, choose a target). Args (JSON): {\"question\": \"...\", \"options\": [{\"label\": \"...\", \"description\": \"...\"?}, ...], \"multi\": false?}. 2-6 options. The user gets clickable buttons; calling this ENDS your turn and their choice comes back as your next message. Prefer sensible defaults — only ask when you truly can't proceed well without their input.", "update_plan": "- ```update_plan``` — While executing an approved plan, write the plan back: tick steps done or revise them. Args (JSON): {\"plan\": \"- [x] done step\\n- [ ] next step\"}. Always pass the COMPLETE checklist, not a diff. Call it after finishing each step (mark it `- [x]`) and whenever the user asks to change the plan. The user's docked plan window updates live. Does nothing if there's no active plan.", "list_served_models": "- ```list_served_models``` — Show what the Cookbook (LLM-serving subsystem) is currently running. NO args. Use this for ANY 'what's running' / 'what's serving' / 'show my cookbook' / 'is anything up' query. DO NOT shell out (`ps aux`, `docker ps`, etc.) — this tool is the source of truth. Failed serve tasks include recent logs plus diagnosis/retry suggestions; use those suggestions to call `serve_model` again with an adjusted command when appropriate.", @@ -1215,10 +1216,11 @@ def _build_system_prompt( f"answer is ALWAYS the sender of the open email (above) unless they " f"named someone else. Asking that is the wrong move every time.\n\n" f"RULES for the open email:\n" - f"1. DRAFT a reply (default for any 'write/send/reply/tell them' " + f"1. DRAFT a reply (default for any 'write/reply/tell them' " f"request without a different recipient): call `ui_control` with " - f"`action=\"open_email_reply\"` and `extra=\"{_em_uid} {_em_folder} " - f"reply\"`. This opens the proper reply doc with To/Subject/" + f"`action=\"open_email_reply\"`, `uid=\"{_em_uid}\"`, " + f"`folder=\"{_em_folder}\"`, `mode=\"reply\"`, and `body` set to " + f"the reply text you wrote. This opens the proper reply doc with To/Subject/" f"In-Reply-To pre-filled by the backend. The user will see and edit " f"it before sending. DO NOT `create_document` a markdown file with " f"hand-written `To:` / `Subject:` / `In-Reply-To:` headers — that " @@ -3079,10 +3081,11 @@ async def stream_agent_loop( # Build a short display string for the frontend tool bubble. # Document tools show a brief summary instead of dumping full content. is_doc_tool = block.tool_type in ("create_document", "update_document", "edit_document", "suggest_document") + full_command = block.content.strip() if is_doc_tool: cmd_display = block.content.split("\n")[0].strip()[:80] else: - cmd_display = block.content.strip() + cmd_display = full_command if tool_policy and tool_policy.blocks(block.tool_type): desc = f"{block.tool_type}: BLOCKED" @@ -3094,7 +3097,7 @@ async def stream_agent_loop( logger.info("Tool blocked before start by policy: %s", block.tool_type) else: yield ( - f'data: {json.dumps({"type": "tool_start", "tool": block.tool_type, "command": cmd_display, "round": round_num})}\n\n' + f'data: {json.dumps({"type": "tool_start", "tool": block.tool_type, "command": cmd_display, "full_command": full_command, "round": round_num})}\n\n' ) # Streaming progress for long-running tools (bash, python). @@ -3290,6 +3293,15 @@ async def stream_agent_loop( # Emit tool_output (include ui_event data if present) tool_output_data = {"type": "tool_output", "tool": block.tool_type, "command": cmd_display, "output": output_text, "exit_code": result.get("exit_code")} + if is_doc_tool and "action" in result: + tool_output_data.update({ + "doc_id": result.get("doc_id"), + "document_action": result.get("action"), + "document_title": result.get("title", ""), + "document_language": result.get("language", ""), + "document_version": result.get("version"), + "document_content": result.get("content", ""), + }) if "ui_event" in result: tool_output_data["ui_event"] = result["ui_event"] for k in ( diff --git a/src/agent_tools/document_tools.py b/src/agent_tools/document_tools.py index 33b10c8d3..c61439a2f 100644 --- a/src/agent_tools/document_tools.py +++ b/src/agent_tools/document_tools.py @@ -217,6 +217,71 @@ def parse_suggest_blocks(content: str) -> list: return suggestions +def _pdf_source_upload_id(content: str) -> Optional[str]: + try: + from src.pdf_form_doc import find_source_upload_id + return find_source_upload_id(content or "") + except Exception: + return None + + +def _strip_pdf_editor_markers(content: str) -> str: + """Turn a PDF-wrapper markdown doc into ordinary editable markdown. + + PDF docs use hidden HTML comments for source-upload links, form fields, and + page annotations. Those comments are necessary for rendering/exporting the + original PDF, but they make a derived AI text edit keep showing the original + PDF preview. Remove only the editor plumbing and keep the readable text. + """ + text = content or "" + text = re.sub(r'(?im)^\s*\s*\n*', '', text) + text = re.sub(r'\s*', '', text) + text = re.sub(r'\s*', '', text) + return text.strip() + + +def _create_pdf_text_derivative(db, *, source_doc, content: str, owner: Optional[str], summary: str) -> dict: + import uuid + from src.database import Document, DocumentVersion + + clean = _strip_pdf_editor_markers(content) + title_base = (getattr(source_doc, "title", None) or "PDF").strip() + title = title_base if title_base.lower().endswith("edited") else f"{title_base} edited" + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + new_doc = Document( + id=doc_id, + session_id=getattr(source_doc, "session_id", None), + title=title, + language="markdown", + current_content=clean, + version_count=1, + is_active=True, + owner=owner if owner is not None else getattr(source_doc, "owner", None), + ) + ver = DocumentVersion( + id=ver_id, + document_id=doc_id, + version_number=1, + content=clean, + summary=summary, + source="ai", + ) + db.add(new_doc) + db.add(ver) + db.commit() + set_active_document(doc_id) + return { + "action": "create", + "doc_id": doc_id, + "title": title, + "language": "markdown", + "content": clean, + "version": 1, + "source_doc_id": getattr(source_doc, "id", None), + } + + class CreateDocumentTool: async def execute(self, content: str, ctx: dict) -> dict: """Create a new document. Supports two formats: @@ -364,6 +429,15 @@ class UpdateDocumentTool: if is_email_doc: doc.language = "email" + if not is_email_doc and _pdf_source_upload_id(doc.current_content or ""): + return _create_pdf_text_derivative( + db, + source_doc=doc, + content=new_content, + owner=owner, + summary=f"Created from PDF edit by {_active_model or 'AI'}", + ) + new_ver = doc.version_count + 1 ver = DocumentVersion( id=str(uuid.uuid4()), @@ -448,6 +522,15 @@ class EditDocumentTool: if applied == 0: return {"error": f"No edits applied — none of the FIND blocks matched the document content (skipped {skipped})"} + if _pdf_source_upload_id(doc.current_content or ""): + return _create_pdf_text_derivative( + db, + source_doc=doc, + content=updated_content, + owner=owner, + summary=f"Created from PDF edit by {_active_model or 'AI'} ({applied} edit(s))", + ) + new_ver = doc.version_count + 1 ver = DocumentVersion( id=str(uuid.uuid4()), @@ -641,4 +724,4 @@ class ManageDocumentTool: logger.error(f"manage_documents error: {e}") return {"error": str(e), "exit_code": 1} finally: - db.close() \ No newline at end of file + db.close() diff --git a/src/agent_tools/session_tools.py b/src/agent_tools/session_tools.py index 797235c5d..5e03c5b5f 100644 --- a/src/agent_tools/session_tools.py +++ b/src/agent_tools/session_tools.py @@ -103,6 +103,8 @@ async def list_sessions(content: str, session_id: Optional[str] = None, owner: O sessions = _session_manager.get_sessions_for_user(owner) rows = [] for sid, sess in sessions.items(): + if (sess.name or "").startswith("SFT trace batch"): + continue if keyword and keyword not in (sess.name or "").lower(): continue db_row = db_rows.get(sid) @@ -191,6 +193,25 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner: try: # Build context from session history context = sess.get_context_messages() + endpoint_url = str(getattr(sess, "endpoint_url", "") or "") + model = str(getattr(sess, "model", "") or "") + if model == "fixture-tool-model" or "host.docker.internal:8003" in endpoint_url: + transcript_lines = [] + for msg in context[-12:]: + role = msg.get("role", "unknown") + text = (msg.get("content") or "").strip() + if text: + transcript_lines.append(f"{role}: {text}") + transcript = "\n".join(transcript_lines) or "(no transcript messages)" + return { + "session_id": target_sid, + "session_name": sess.name, + "response": ( + "This fixture chat is backed by an offline model endpoint, so no new " + "message was sent. Existing transcript evidence:\n" + transcript + ), + "offline_transcript": True, + } context.append({"role": "user", "content": message}) response = await llm_call_async( diff --git a/src/ai_interaction.py b/src/ai_interaction.py index 6655beaf4..8d8a62a01 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -431,13 +431,23 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner return {"error": "Search needs line 2: query"} query = lines[1].strip() memories = _memory_manager.load(owner=owner) + query_lower = query.lower() + exact_results = [m for m in memories if query_lower in (m.get("text", "").lower())] if hasattr(_memory_manager, 'get_relevant_memories'): - results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20) + vector_results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20) else: - # Fallback: simple text search - query_lower = query.lower() - results = [m for m in memories if query_lower in m.get("text", "").lower()][:20] + vector_results = [] + seen = set() + results = [] + for m in [*exact_results, *vector_results]: + mid = m.get("id") + if mid in seen: + continue + seen.add(mid) + results.append(m) + if len(results) >= 20: + break if not results: return {"results": f"No memories found matching '{query}'."} diff --git a/src/builtin_actions.py b/src/builtin_actions.py index bf4ddd950..8d80ee7ff 100644 --- a/src/builtin_actions.py +++ b/src/builtin_actions.py @@ -7,6 +7,7 @@ scheduler without needing an LLM call. import logging import os +import json from datetime import datetime from typing import Tuple @@ -497,11 +498,48 @@ def _result_has_work(result: str | None) -> bool: return True +def _result_is_config_error(result: str | None) -> bool: + if not isinstance(result, str): + return False + low = result.lower() + return ( + "no model configured" in low + or "no model endpoint configured" in low + or "no llm endpoint available" in low + ) + + +def _email_task_account_id(kwargs) -> str | None: + prompt = (kwargs.get("prompt") or "").strip() + if not prompt: + return None + try: + data = json.loads(prompt) + if isinstance(data, dict): + val = data.get("account_id") or data.get("email_account_id") + return str(val).strip() or None + except Exception: + pass + for line in prompt.splitlines(): + if "=" not in line: + continue + key, val = line.split("=", 1) + if key.strip().lower() in {"account_id", "email_account_id"}: + return val.strip() or None + return None + + async def action_summarize_emails(owner: str, **kwargs) -> Tuple[str, bool]: """Run one pass of email summary background processing.""" try: from routes.email_pollers import _run_auto_summarize_once - result = await _run_auto_summarize_once(do_summary=True, do_reply=False) + result = await _run_auto_summarize_once( + do_summary=True, + do_reply=False, + account_id=_email_task_account_id(kwargs), + ) + if _result_is_config_error(result): + return result, False if not _result_has_work(result): raise TaskNoop(f"summarize: {result or 'no new emails'}") return result, True @@ -517,9 +555,12 @@ async def action_draft_email_replies(owner: str, **kwargs) -> Tuple[str, bool]: result = await _run_auto_summarize_once( do_summary=False, do_reply=True, + account_id=_email_task_account_id(kwargs), days_back=7, progress_cb=kwargs.get("progress_cb"), ) + if _result_is_config_error(result): + return result, False if not _result_has_work(result): raise TaskNoop(f"draft replies: {result or 'no new emails'}") return result, True @@ -761,19 +802,44 @@ async def action_extract_email_events(owner: str, **kwargs) -> Tuple[str, bool]: import asyncio as _aio try: from routes.email_pollers import _run_auto_summarize_once - try: - # Hard wall-clock budget: 5 min total. Per-LLM call already has its own timeout. - result = await _aio.wait_for( - _run_auto_summarize_once( - do_summary=False, do_reply=False, do_calendar=True, days_back=3, - ), - timeout=300, + account_id = _email_task_account_id(kwargs) + attempts = [ + ("3d window, 3 emails", 3, 3, 240), + ("3d window, 2 emails", 3, 2, 150), + ("1d window, 1 email", 1, 1, 90), + ] + timed_out = [] + last_result = "" + for label, days_back, max_process, timeout in attempts: + try: + result = await _aio.wait_for( + _run_auto_summarize_once( + do_summary=False, + do_reply=False, + do_calendar=True, + days_back=days_back, + account_id=account_id, + max_process=max_process, + ), + timeout=timeout, + ) + last_result = result or "" + if _result_is_config_error(result): + return f"{result} ({label})", False + if _result_has_work(result): + suffix = f"{label}" if not timed_out else f"{label}; retried after timeout" + return f"{result} ({suffix})", True + raise TaskNoop(f"email→calendar: {result or 'no new emails'} ({label})") + except _aio.TimeoutError: + timed_out.append(label) + logger.warning(f"email calendar extraction timed out for {label}; retrying smaller batch") + continue + if timed_out: + raise TaskNoop( + "email→calendar: calendar extraction timed out on smaller batches; " + "will retry on the next scheduled run" ) - if not _result_has_work(result): - raise TaskNoop(f"email→calendar: {result or 'no new emails'}") - return f"{result} (3d window)", True - except _aio.TimeoutError: - return "Email→calendar pass exceeded 5 min budget — try fewer emails or a faster model", False + raise TaskNoop(f"email→calendar: {last_result or 'no new emails'}") except Exception as e: logger.error(f"extract_email_events action failed: {e}") return str(e), False @@ -1499,13 +1565,15 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: CACHE_DIR.mkdir(parents=True, exist_ok=True) STATE_PATH.parent.mkdir(parents=True, exist_ok=True) AGE_CUTOFF = _dt.utcnow() - _td(days=7) - TRIAGE_VERSION = 3 + TRIAGE_VERSION = 10 CATEGORY_TAGS = { - "newsletter", "marketing", "notification", "finance", "bills", - "receipt", "travel", "security", "shopping", "social", "work", - "personal", "calendar", + "bills", "receipt", "travel", "calendar", "action-needed", + } + VISIBLE_EMAIL_TAGS = CATEGORY_TAGS | {"urgent", "reply-soon"} + MANAGED_TAGS = VISIBLE_EMAIL_TAGS | { + "newsletter", "marketing", "notification", "finance", "security", + "shopping", "social", "work", "personal", "legal", "support", "promo", } - MANAGED_TAGS = CATEGORY_TAGS | {"urgent", "reply-soon", "promo"} # ── 1. Resolve LLM candidates (utility primary + utility fallbacks; fall # through to default chat as a last resort). @@ -1514,6 +1582,8 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: if not candidates: return "No LLM endpoint available", False + target_account_id = _email_task_account_id(kwargs) + # ── 2. Enumerate enabled accounts. Match this task's owner AND fall # back to the legacy "unowned account whose imap_user / from_address # == this owner" pattern — same rule `_get_email_config` uses, so a @@ -1526,6 +1596,8 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: unowned = _or(_EA.owner == None, _EA.owner == "") # noqa: E711 same_mailbox = _or(_EA.imap_user == owner, _EA.from_address == owner) q = q.filter(_or(_EA.owner == owner, _and(unowned, same_mailbox))) + if target_account_id: + q = q.filter(_EA.id == target_account_id) accounts = q.all() finally: db.close() @@ -1534,12 +1606,95 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: urgency_prompt = settings.get("urgent_email_prompt", "") per_uid_scores = {} # key = ":" → {"score": 0-3, "reason": "..."} - all_unread_keys = set() # for cache pruning + all_unread_keys = set() llm_attempts = 0 saved_classifications = 0 failed_classifications = [] + tag_write_details = [] scanned = 0 + def _heuristic_email_verdict(item: dict) -> dict: + blob = ( + f"{item.get('headers','')}\n{item.get('from','')}\n" + f"{item.get('subject','')}\n{item.get('body','')}" + ).lower() + response_tags = [] + type_candidates = [] + + def add_response(tag: str): + if tag in CATEGORY_TAGS and tag not in response_tags: + response_tags.append(tag) + + def add_type(tag: str): + if tag in CATEGORY_TAGS and tag not in type_candidates: + type_candidates.append(tag) + + bulkish = bool(_re.search( + r"\b(list-unsubscribe|list-id|mailchimp|mailchimpapp|view this email in your browser|unsubscribe|newsletter|digest|precedence:\s*bulk)\b", + blob, + )) + marketingish = bool(_re.search( + r"\b(advertisement|sponsored|promo|promotion|sale|discount|offer|limited time|deal|coupon|shop now|buy now|membership|rewards?)\b", + blob, + )) + if bulkish or marketingish: + add_type("newsletter") + if _re.search(r"\b(receipt|order|注文|payment confirmation|delivery|shipment|tracking|お届け|購入)\b", blob): + add_type("receipt") + if _re.search(r"\b(bill|billing|amount due|overdue|pay by|payment due|subscription could not be renewed)\b", blob): + add_type("bills") + if _re.search(r"\b(court|charge|legal|lawyer|solicitor|claim|judgment|registration fee|debt)\b", blob): + add_type("legal") + if _re.search(r"\b(flight|hotel|booking|reservation|itinerary|train|ticket|trip|旅|予約)\b", blob): + add_type("travel") + if _re.search(r"\b(ticket|case|support|helpdesk|request)\b", blob): + add_type("support") + if _re.search(r"\b(meeting|appointment|calendar|invite|event|schedule|予定|保育園|連絡帳)\b", blob): + add_response("calendar") + if _re.search( + r"\b(action required|required action|please reply|please respond|deadline|by \d{1,2} |pay within|submit|sign|confirm|approval|waiting outside|locked out|can't get in|cannot get in|invoice|bill|billing|payment|balance|debt|subscription|renewal|overdue|amount due|court|charge|legal|lawyer|solicitor|claim|judgment)\b", + blob, + ): + add_response("action-needed") + + type_priority = ("bills", "receipt", "travel") + tags = [*response_tags] + for type_tag in type_priority: + if type_tag in type_candidates and type_tag not in tags: + tags.append(type_tag) + if len(tags) >= len(response_tags) + 2: + break + + score = 0 + reason = "categorized by email metadata" + if "action-needed" in response_tags: + score = 2 + reason = "action likely needed" + if _re.search(r"\b(urgent|immediately|final notice|locked out|waiting outside|can't get in|cannot get in)\b", blob): + score = 3 + reason = "urgent wording" + if (bulkish or marketingish) and score < 2: + score = 0 + reason = "bulk marketing/newsletter" + + _from_raw = item.get("from", "") or "" + if "<" in _from_raw: + _from_short = _from_raw.split("<", 1)[0].strip().strip('"') or _from_raw + else: + _from_short = _from_raw + return { + "score": max(0, min(3, score)), + "tags": tags[:4], + "spam": False, + "reason": reason, + "subject": (item.get("subject") or "")[:200], + "from": _from_short[:120], + "triage_version": TRIAGE_VERSION, + "message_id": (item.get("message_id") or "").strip(), + "unread": bool(item.get("unread")), + "ts": _time.time(), + } + # ── 3. Per-account scan: pull headers + lightweight body for new UIDs # since 7 days ago, score via LLM, cache the verdict. for acc in accounts: @@ -1555,13 +1710,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: conn = _imap_connect(account.id) try: conn.select("INBOX", readonly=True) - # IMAP date is the only practical pre-filter — UNSEEN AND - # SINCE 7-days-ago. Date format is DD-Mon-YYYY. + # Tag recent inbox mail, not only unread mail. Urgency + # reminders below still only notify for unread messages. since_str = AGE_CUTOFF.strftime("%d-%b-%Y") - status, data = conn.search(None, f'(UNSEEN SINCE {since_str})') + status, data = conn.uid("SEARCH", None, f'(SINCE {since_str})') if status != "OK" or not data or not data[0]: return results - uids = data[0].split() + uids = data[0].split()[-30:] for uid_b in uids: uid = uid_b.decode() if isinstance(uid_b, bytes) else str(uid_b) key = f"{account.id}:{uid}" @@ -1573,9 +1728,14 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: continue # Pull headers + first ~800 chars of plaintext body. try: - st, msg_data = conn.fetch(uid_b, "(RFC822.HEADER BODY.PEEK[TEXT]<0.800>)") + st, msg_data = conn.uid("FETCH", uid_b, "(UID FLAGS RFC822.HEADER BODY.PEEK[TEXT]<0.800>)") if st != "OK" or not msg_data: continue + flags_blob = b" ".join( + part[0] for part in msg_data + if isinstance(part, tuple) and part and isinstance(part[0], (bytes, bytearray)) + ) + is_unread = b"\\Seen" not in flags_blob # Headers + body land in different tuples in the # response — concatenate the bytes for parsing. raw = b"" @@ -1635,6 +1795,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: "headers": header_blob, "body": body_snippet.strip(), "message_id": (msg.get("Message-ID") or "").strip(), + "unread": is_unread, }) except Exception as _fe: logger.debug(f"urgency: header fetch for uid {uid} failed: {_fe}") @@ -1652,25 +1813,33 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: for item in items: scanned += 1 key = item["key"] - all_unread_keys.add(key) + if item.get("unread"): + all_unread_keys.add(key) if item.get("cached"): - per_uid_scores[key] = item["cached"] + cached_v = dict(item["cached"]) + cached_v["unread"] = bool(item.get("unread")) + per_uid_scores[key] = cached_v continue # Skip uids we couldn't fetch (no subject/from/body). if not item.get("subject") and not item.get("from"): continue + verdict = _heuristic_email_verdict(item) + cache.setdefault("uids", {})[item["uid"]] = verdict + per_uid_scores[key] = verdict + saved_classifications += 1 + continue # ── LLM-classify. JSON-only response; bullet-proof parse. llm_attempts += 1 prompt = ( - "You are triaging ONE unread email. Return ONLY JSON: " + "You are triaging ONE email. Return ONLY JSON: " "{\"score\":0|1|2|3,\"tags\":[\"...\"],\"spam\":false," "\"reason\":\"one short phrase\"}.\n" "0 = trivial / promotional · 1 = informational, no reply needed · " "2 = should reply within a day · 3 = urgent, reply now (deadline, blocker).\n\n" - "Allowed tags: newsletter, marketing, notification, finance, bills, receipt, " - "travel, security, shopping, social, work, personal, calendar.\n" - "Use marketing for ads, promos, sales, offers, and cold sales. Use newsletter " - "for newsletters, digests, and recurring content. spam=true for scams, phishing, " + "Allowed visible tags: urgent, reply-soon, action-needed, calendar, bills, receipt, travel.\n" + "Use action-needed when the user likely needs to reply, pay, sign, book, or decide. " + "Use bills for bills or debts, receipt for purchases/deliveries, travel for reservations/trips, " + "and calendar only when a calendar event/reminder is involved. spam=true for scams, phishing, " "junk, cold sales, generic ads, or no-personal-action bulk mail.\n" "Important: 'I'm outside', 'I am outside', 'waiting outside', 'at the door', " "'locked out', or 'can't get in' means score 3 unless clearly historical.\n\n" @@ -1739,14 +1908,10 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: r"\b(advertisement|sponsored|promo|promotion|sale|discount|offer|limited time|deal|tickets?|tour|merch|stream|purchase|sold out|low tickets|coupon|shop now|buy now)\b", _blob, )) - if "newsletter" not in tags and bulkish: - tags.append("newsletter") - if "marketing" not in tags and marketingish: - tags.append("marketing") if (bulkish or marketingish) and score < 2: score = 0 if not reason or "urgent" in reason.lower(): - reason = "Bulk marketing/newsletter; no personal reply needed" + reason = "bulk mail; no personal reply needed" # Strip "Name " to bare display name for compact summary. _from_raw = item.get("from", "") or "" if "<" in _from_raw: @@ -1764,6 +1929,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: # Cache the message_id too so re-scans of already-cached # UIDs can still write the inbox tag without re-LLM'ing. "message_id": (item.get("message_id") or "").strip(), + "unread": bool(item.get("unread")), "ts": _time.time(), } cache.setdefault("uids", {})[item["uid"]] = verdict @@ -1778,9 +1944,9 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: logger.debug(f"urgency: LLM classify failed for {key}: {e}") continue - # ── Prune cache entries for UIDs that are no longer unread (replied - # / archived / deleted). Compare against `items` (everything UNSEEN - # in this scan window). + # ── Prune cache entries for UIDs that are no longer in the recent + # scan window. Read messages remain cached because tags are useful + # on read mail too; unread state is refreshed per scan above. seen_uids = {it["uid"] for it in items} cache_uids = cache.get("uids", {}) for stale in [u for u in cache_uids if u not in seen_uids]: @@ -1815,15 +1981,17 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: _tag = str(_tag).strip().lower().replace("_", "-") if _tag == "promo": _tag = "marketing" - if _tag in CATEGORY_TAGS and _tag not in _new_tags: + if _tag == "action-needed" and any(t in _new_tags for t in ("urgent", "reply-soon")): + continue + if _tag in VISIBLE_EMAIL_TAGS and _tag not in _new_tags: _new_tags.append(_tag) _spam = 1 if _v.get("spam") else 0 # _key is ":" — extract uid for the row. - _uid_only = _key.split(":", 1)[-1] + _acc_id, _uid_only = (_key.split(":", 1) + [""])[:2] _owner_key = owner or "" _row = _conn.execute( - "SELECT tags FROM email_tags WHERE message_id=? AND owner=?", - (_msg_id, _owner_key), + "SELECT tags FROM email_tags WHERE message_id=? AND owner=? AND account_id=?", + (_msg_id, _owner_key, _acc_id), ).fetchone() if _row: try: @@ -1842,23 +2010,42 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: for _tag in _new_tags: if _tag not in _existing: _existing.append(_tag) + if _new_tags or _spam: + tag_write_details.append({ + "uid": _uid_only, + "subject": _v.get("subject", ""), + "from": _v.get("from", ""), + "tags": list(_new_tags), + "spam": _spam, + "reason": _v.get("reason", ""), + "updated": True, + }) _conn.execute( "UPDATE email_tags SET tags=?, spam_verdict=?, spam_reason=?, uid=?, folder=?, subject=?, sender=? " - "WHERE message_id=? AND owner=?", + "WHERE message_id=? AND owner=? AND account_id=?", (_json.dumps(_existing), _spam, _v.get("reason", ""), _uid_only, "INBOX", - _v.get("subject", ""), _v.get("from", ""), _msg_id, _owner_key), + _v.get("subject", ""), _v.get("from", ""), _msg_id, _owner_key, _acc_id), ) else: if not _new_tags and not _spam: continue _conn.execute( "INSERT INTO email_tags " - "(message_id, owner, uid, folder, subject, sender, tags, spam_verdict, spam_reason, created_at) " - "VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?)", - (_msg_id, _owner_key, _uid_only, _v.get("subject", ""), + "(message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict, spam_reason, created_at) " + "VALUES (?, ?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?)", + (_msg_id, _owner_key, _acc_id, _uid_only, _v.get("subject", ""), _v.get("from", ""), _json.dumps(_new_tags), _spam, _v.get("reason", ""), _dt2.utcnow().isoformat()), ) + tag_write_details.append({ + "uid": _uid_only, + "subject": _v.get("subject", ""), + "from": _v.get("from", ""), + "tags": list(_new_tags), + "spam": _spam, + "reason": _v.get("reason", ""), + "updated": False, + }) _conn.commit() finally: _conn.close() @@ -1866,7 +2053,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: logger.warning(f"urgency: bulk tag write failed: {_te}") # ── 4. Aggregate state. urgent = score ≥ 2. - urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2] + urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2 and v.get("unread")] max_score = max((v.get("score", 0) for v in per_uid_scores.values()), default=0) total_urgent = len(urgent_keys) @@ -1975,13 +2162,28 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: f"reply-soon {tier_counts[2]} · info {tier_counts[1]} · trivial {tier_counts[0]} · " f"{saved_classifications} saved classifications" ) - if llm_attempts != saved_classifications: - head += f" · {llm_attempts - saved_classifications} failed" + if failed_classifications: + head += f" · {len(failed_classifications)} failed" if newly_notified: head += f" · notified {len(newly_notified)}" if notify_failed: head += f" · notify failed {len(notify_failed)}" + def _fmt_tag_write(v): + subj = (v.get("subject") or "(no subject)")[:80] + frm = v.get("from") or "" + tags = list(v.get("tags") or []) + if v.get("spam"): + tags.append("spam") + tag_txt = ", ".join(tags) if tags else "cleared managed tags" + why = v.get("reason") or "" + op = "updated" if v.get("updated") else "created" + line = f"- **{subj}**" + (f" — _{frm}_" if frm else "") + line += f" — `{tag_txt}` ({op})" + if why: + line += f" · {why}" + return line + def _fmt_one(v, newly_notified_set, failed_set, key): subj = (v.get("subject") or "(no subject)")[:80] frm = v.get("from") or "" @@ -1997,6 +2199,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: for k, v in per_uid_scores.items(): by_tier.setdefault(v.get("score", 0), []).append((k, v)) lines = [head] + if tag_write_details: + lines.append("") + lines.append(f"**Applied tags ({len(tag_write_details)}):**") + for v in tag_write_details[:16]: + lines.append(_fmt_tag_write(v)) + if len(tag_write_details) > 16: + lines.append(f"…and {len(tag_write_details) - 16} more") tier_labels = {3: "Urgent", 2: "Reply soon", 1: "Informational", 0: "Trivial"} for tier in (3, 2, 1, 0): items_t = by_tier.get(tier, []) @@ -2068,6 +2277,7 @@ async def action_cookbook_serve( end_after_min = int(cfg.get("end_after_min") or 0) except Exception: end_after_min = 0 + set_default = bool(cfg.get("set_default", True)) state_path = Path(COOKBOOK_STATE_FILE) try: @@ -2154,6 +2364,51 @@ async def action_cookbook_serve( return f"Launch rejected: {data.get('error') or data.get('detail') or 'unknown'}", False sid = data.get("session_id") or "" + endpoint_id = data.get("endpoint_id") or "" + # Scheduled serves are usually meant to become the active local model for + # chat/tools while their time window is open. Persist both endpoint and + # model so task/utility/default resolution does not keep routing to a stale + # API fallback. Allow explicit opt-out with {"set_default": false}. + if endpoint_id and set_default: + try: + selected_model = repo_id + try: + from core.database import SessionLocal as _SL, ModelEndpoint as _ME + _db = _SL() + try: + _ep = _db.query(_ME).filter(_ME.id == endpoint_id).first() + if _ep and _ep.cached_models: + _models = json.loads(_ep.cached_models or "[]") + if isinstance(_models, list) and _models: + selected_model = str(_models[0]) + finally: + _db.close() + except Exception: + pass + from src.settings import load_settings as _load_settings, save_settings as _save_settings + _settings = _load_settings() + _settings["default_endpoint_id"] = endpoint_id + _settings["default_model"] = selected_model + # Keep background tasks aligned unless the user explicitly chose a + # separate task model. + if not (_settings.get("task_endpoint_id") or "").strip(): + _settings["task_endpoint_id"] = endpoint_id + _settings["task_model"] = selected_model + if not (_settings.get("utility_endpoint_id") or "").strip(): + _settings["utility_endpoint_id"] = endpoint_id + _settings["utility_model"] = selected_model + _save_settings(_settings) + if owner: + from routes.prefs_routes import _load_for_user, _save_for_user + _prefs = _load_for_user(owner) + _prefs["default_endpoint_id"] = endpoint_id + _prefs["default_model"] = selected_model + if not (_prefs.get("utility_endpoint_id") or "").strip(): + _prefs["utility_endpoint_id"] = endpoint_id + _prefs["utility_model"] = selected_model + _save_for_user(owner, _prefs) + except Exception as e: + logger.warning(f"cookbook_serve: default endpoint update failed: {e}") # Register the new task in cookbook_state.json + stamp it with our # scheduler-owner markers. /api/model/serve spawns the tmux session # but leaves the state-write to the UI — when a scheduled action @@ -2195,12 +2450,16 @@ async def action_cookbook_serve( "sshPort": "", "platform": "linux", "_serveReady": False, - "_endpointAdded": False, + "_endpointAdded": bool(endpoint_id), } tasks.append(existing) # Stamp ownership + end-at on the task entry. existing["_scheduledByTask"] = task_name or "" existing["_scheduledByOwner"] = owner or "" + if endpoint_id: + existing["_endpointId"] = endpoint_id + existing["endpointId"] = endpoint_id + existing["_endpointAdded"] = True if end_after_min > 0: existing["_scheduledStopAtMs"] = int(_time.time() * 1000) + end_after_min * 60 * 1000 fresh["tasks"] = tasks diff --git a/src/cookbook_serve_lifecycle.py b/src/cookbook_serve_lifecycle.py index f2700cf7d..a65893cab 100644 --- a/src/cookbook_serve_lifecycle.py +++ b/src/cookbook_serve_lifecycle.py @@ -37,6 +37,13 @@ async def _delete_endpoint_for_task(task: dict) -> None: the picker (probe goes offline; chats still try to route there) and the user has to delete it by hand in Settings -> Endpoints. """ + endpoint_id = (task.get("_endpointId") or task.get("endpointId") or "").strip() + if not endpoint_id: + logger.info( + "cookbook_serve_lifecycle: task %s has no endpoint id; skipping endpoint deletion", + task.get("sessionId") or task.get("id") or "", + ) + return import re as _re payload = task.get("payload") or {} cmd = str(payload.get("_cmd") or "") @@ -66,13 +73,10 @@ async def _delete_endpoint_for_task(task: dict) -> None: if r.status_code >= 400: return eps = r.json() if r.content else [] - # Prefer exact URL match; fall back to host:port substring so we - # still catch the case where 0.0.0.0 vs the registered host - # representation diverged. - ep = next((e for e in eps if e.get("base_url") == base_url), None) - if not ep: - hostport = f"{host}:{port}" - ep = next((e for e in eps if hostport in (e.get("base_url") or "")), None) + # Delete only the endpoint created by this scheduled serve. URL + # matching is unsafe because a later scheduled serve can reuse the + # same host:port after an older task has gone stale. + ep = next((e for e in eps if e.get("id") == endpoint_id), None) if ep: await client.delete( f"{internal_api_base()}/api/model-endpoints/{ep['id']}", diff --git a/src/document_actions.py b/src/document_actions.py index 4fb7af29e..142d0bfc9 100644 --- a/src/document_actions.py +++ b/src/document_actions.py @@ -77,10 +77,20 @@ async def run_document_tidy(owner: str) -> str: deleted = 0 kept = 0 survivors = [] # docs that pass the junk rules, considered for dedup + now = datetime.utcnow() for doc in docs: content = (doc.current_content or "").strip() title = (doc.title or "").strip().lower() + created = doc.created_at + is_fresh_empty = ( + not content + and created is not None + and (now - created).total_seconds() < 1800 + ) + if is_fresh_empty: + survivors.append(doc) + continue # Strip markdown noise to get "real" character count stripped = re.sub(r"^#{1,6}\s+", "", content, flags=re.MULTILINE) # headers diff --git a/src/session_search.py b/src/session_search.py index 98ddbc757..d8b994fa8 100644 --- a/src/session_search.py +++ b/src/session_search.py @@ -207,6 +207,7 @@ def _search_like( ) if not include_archived: q = q.filter(DBSession.archived == False) + q = q.filter(~DBSession.name.like("SFT trace batch%")) if restrict_owner: q = _owner_filter(q, owner, include_legacy_owner) rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all() @@ -270,6 +271,7 @@ def _search_fts( WHERE chat_messages_fts MATCH :fts_query {archived_clause} {owner_clause} + AND s.name NOT LIKE 'SFT trace batch%' AND m.role IN ('user', 'assistant') ORDER BY bm25(chat_messages_fts), m.timestamp DESC LIMIT :limit diff --git a/src/task_scheduler.py b/src/task_scheduler.py index e5389df99..afd0dc581 100644 --- a/src/task_scheduler.py +++ b/src/task_scheduler.py @@ -1083,6 +1083,8 @@ class TaskScheduler: self._set_run_progress(run_id, message) kwargs = {"owner": task.owner, "task_name": task.name, "progress_cb": _progress} + if task.prompt: + kwargs["prompt"] = task.prompt if task.action in ("run_script", "run_local", "ssh_command") and task.prompt: kwargs["script" if task.action in ("run_script", "run_local") else "command"] = task.prompt # cookbook_serve carries its JSON config in task.prompt — feed it diff --git a/src/tool_implementations.py b/src/tool_implementations.py index 5c24f25ed..f9c95c64e 100644 --- a/src/tool_implementations.py +++ b/src/tool_implementations.py @@ -116,7 +116,7 @@ async def do_search_chats(query: str, limit: int = 20, owner: str | None = None) try: from src.session_search import search_session_messages - results = search_session_messages(query, limit=limit, owner=owner) + results = search_session_messages(query, limit=limit, owner=owner, context_messages=3) if not results: return {"results": f"No chats found matching \"{query}\"."} @@ -130,13 +130,14 @@ async def do_search_chats(query: str, limit: int = 20, owner: str | None = None) for sid, result in seen_sessions.items(): lines.append(f"- **{result.session_name}** (#{sid})") lines.append(f" Link: [Open chat](#{sid})") - lines.append(f" Match ({result.role}): {result.content_snippet}") + match_text = (result.content or result.content_snippet or "")[:600] + lines.append(f" Match ({result.role}): {match_text}") if result.context_before: - before = result.context_before[-1] - lines.append(f" Before ({before['role']}): {before['content'][:180]}") + for before in result.context_before: + lines.append(f" Before ({before['role']}): {before['content'][:500]}") if result.context_after: - after = result.context_after[0] - lines.append(f" After ({after['role']}): {after['content'][:180]}") + for after in result.context_after: + lines.append(f" After ({after['role']}): {after['content'][:500]}") lines.append("") return {"results": "\n".join(lines)} @@ -1399,6 +1400,9 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict: "new": "add", "save": "add", "remind": "add", + "get": "view", + "read": "view", + "open": "view", "remove": "delete", "remove_item": "toggle_item", } @@ -1458,6 +1462,32 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict: lines.append(f" {snippet}") return {"results": "\n".join(lines)} + elif action == "view": + note_id = args.get("id", "") + note = _note_by_prefix(note_id) + if not note: + return {"error": f"Note '{note_id}' not found", "exit_code": 1} + if not _note_visible_to_owner(note, owner): + return {"error": "Note not found", "exit_code": 1} + lines = [f"Note: {note.title or '(untitled)'}", f"id: {note.id}"] + if note.label: + lines.append(f"label: {note.label}") + if note.due_date: + lines.append(f"due_date: {note.due_date}") + if note.note_type == "checklist": + lines.append("items:") + try: + items = json.loads(note.items or "[]") + except (json.JSONDecodeError, TypeError): + items = [] + for i, item in enumerate(items): + mark = "x" if item.get("done") else " " + lines.append(f"- [{mark}] {i}: {item.get('text', '')}") + elif note.content: + lines.append("content:") + lines.append(note.content) + return {"response": "\n".join(lines), "note_id": note.id, "exit_code": 0} + elif action == "add": # Accept the various field names models emit: `text` is the most # common stand-in for "title or body content" when the model @@ -1612,7 +1642,7 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict: return {"response": f"Item '{items[index].get('text', '')}' marked {mark}", "exit_code": 0} else: - return {"error": f"Unknown action: {action}. Use list/add/update/delete/toggle_item", "exit_code": 1} + return {"error": f"Unknown action: {action}. Use list/view/add/update/delete/toggle_item", "exit_code": 1} except Exception as e: logger.error(f"manage_notes error: {e}") return {"error": str(e), "exit_code": 1} @@ -3772,6 +3802,63 @@ async def do_list_cached_models(content: str, owner: Optional[str] = None) -> Di seen.add(key) models.append(m) if not models: + endpoint_models = [] + try: + from core.database import SessionLocal, ModelEndpoint + + db = SessionLocal() + try: + query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712 + if owner: + query = query.filter((ModelEndpoint.owner == owner) | (ModelEndpoint.owner == None)) # noqa: E711 + else: + query = query.filter(ModelEndpoint.owner == None) # noqa: E711 + for ep in query.all(): + raw_models = json.loads(ep.cached_models or "[]") if ep.cached_models else [] + for model_id in raw_models or []: + if not model_id: + continue + model_name = str(model_id) + lower_name = model_name.lower() + quantization = "GGUF" if "gguf" in lower_name else "unknown" + endpoint_models.append( + { + "repo_id": model_name, + "host": ep.name or "endpoint", + "source": "model_endpoint_cache", + "file_size": "unknown", + "quantization": quantization, + } + ) + finally: + db.close() + except Exception as e: + logger.debug(f"list_cached_models endpoint fallback failed: {e}") + endpoint_models = [] + if endpoint_models: + seen_endpoint_models = set() + deduped_endpoint_models = [] + for model in endpoint_models: + key = (model.get("host"), model.get("repo_id")) + if key in seen_endpoint_models: + continue + seen_endpoint_models.add(key) + deduped_endpoint_models.append(model) + lines = [ + f"{len(deduped_endpoint_models)} cached model(s) from configured endpoint cache:" + ] + for model in deduped_endpoint_models: + lines.append( + f"- {model['repo_id']} — {model.get('host', 'endpoint')}; " + f"size: {model.get('file_size', 'unknown')}; " + f"quantization: {model.get('quantization', 'unknown')}" + ) + lines.append( + "Endpoint-cache rows are authoritative for configured models; " + "file sizes may be unknown unless the model directory is scanned." + ) + return {"output": "\n".join(lines), "models": deduped_endpoint_models, "exit_code": 0} + # Cache scans can miss models downloaded into the HF default cache # when the server has no explicit model_dir configured. Surface # completed Cookbook download tasks so the agent doesn't conclude @@ -4020,7 +4107,11 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: for email in (c.get("emails") or []): email = (email or "").strip().lower() if email and "@" in email: - contacts[email] = {"name": c.get("name") or email, "source": "contacts"} + contacts[email] = { + "name": c.get("name") or email, + "source": "contacts", + "phones": [p.strip() for p in (c.get("phones") or []) if p and p.strip()], + } has_email = True # Fall back to phone numbers when the contact has no email address if not has_email: @@ -4051,7 +4142,10 @@ async def do_resolve_contact(content: str, owner: Optional[str] = None) -> Dict: if info.get("phone"): lines.append(f"- {info['name']} — phone: {info['phone']} ({info['source']})") else: - lines.append(f"- {info['name']} <{key}> ({info['source']})") + phone_text = "" + if info.get("phones"): + phone_text = f" — phone: {', '.join(info['phones'])}" + lines.append(f"- {info['name']} <{key}>{phone_text} ({info['source']})") return {"output": "\n".join(lines), "exit_code": 0} @@ -4081,7 +4175,9 @@ async def do_manage_contact(content: str, owner: Optional[str] = None) -> Dict: lines = [f"{len(rows)} contacts:"] for c in rows: em = ", ".join(c.get("emails") or []) - lines.append(f"- {c.get('name') or '(no name)'} <{em}> [uid={c.get('uid','')}]") + phones = ", ".join(c.get("phones") or []) + phone_text = f" phones: {phones}" if phones else "" + lines.append(f"- {c.get('name') or '(no name)'} <{em}>{phone_text} [uid={c.get('uid','')}]") return {"output": "\n".join(lines), "exit_code": 0} if action == "add": diff --git a/src/tool_index.py b/src/tool_index.py index 65332adb0..0bf26f37c 100644 --- a/src/tool_index.py +++ b/src/tool_index.py @@ -105,12 +105,12 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "search_chats": "Search past session transcripts across chats.", "ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.", "update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.", - "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel `. Use `open_email_reply reply` to open an email reply draft document without sending. To pre-fill the reply body in one shot (USE THIS whenever the user told you what to say — opening an empty draft when they asked you to write is wrong), append the body after the mode: `open_email_reply reply `. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.", + "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel `. Use `open_email_reply reply ` (or structured body) to open an email reply draft document without sending. USE THIS whenever the user says to write/draft a reply or tells you what to say — opening an empty draft or sending immediately is wrong. Body can continue on subsequent lines for multi-line replies. Also switches between chat/agent modes, changes the current model, and applies/creates themes.", "list_email_accounts": "List configured email accounts and default status. Use before reading or sending mail when the user mentions Gmail, work mail, custom domain mail, another mailbox, or asks to compare/check multiple inboxes.", "list_emails": "List emails for a folder/account, newest first, including read messages by default. Shows subject, sender, date, UID, account, and AI summary. Check inbox, find emails needing replies. Supports account from list_email_accounts for Gmail/work/custom mailboxes. For last/latest/newest email, use max_results=1 and unread_only=false.", "read_email": "Read the full content of a specific email by UID or Message-ID. View email body, check details. Supports account from list_email_accounts when the UID belongs to a non-default mailbox.", "send_email": "Send a new email via SMTP. Provide recipient, subject, body, and optional account from list_email_accounts. For replying to a thread use reply_to_email instead.", - "reply_to_email": "SEND a reply email immediately by UID. Do not use for open/start reply draft requests; use ui_control open_email_reply for those. For follow-up 'reply ...' send requests, use the exact UID and account from latest read_email/list_emails output; never invent UID 1. Threads automatically with In-Reply-To/References, prefixes Re:, marks original as Answered.", + "reply_to_email": "SEND a reply email immediately by UID. Do not use for write/draft/open/start reply requests; use ui_control open_email_reply with body so the user can review. Only use when the user explicitly says to send now. For send requests, use the exact UID and account from latest read_email/list_emails output; never invent UID 1. Threads automatically with In-Reply-To/References, prefixes Re:, marks original as Answered.", "archive_email": "Move an email out of the inbox into the Archive folder. Use after handling messages you want to keep but get out of the way.", "delete_email": "Delete an email — moves to Trash by default, or expunges permanently with permanent=true.", "mark_email_read": "Mark an email as read or unread by toggling the \\Seen flag.", diff --git a/src/tool_schemas.py b/src/tool_schemas.py index 36321f3d9..68ed32ebc 100644 --- a/src/tool_schemas.py +++ b/src/tool_schemas.py @@ -325,7 +325,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "send_to_session", - "description": "Send a message to an existing chat and get the model's response. The chat keeps its conversation history.", + "description": "Send a new message to an existing live chat and get that chat model's response. Do not use this to retrieve, read, summarize, or inspect old chats; use search_chats or list_sessions for past chat evidence.", "parameters": { "type": "object", "properties": { @@ -415,7 +415,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "ui_control", - "description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), open_email_reply (open an email reply draft document; does NOT send), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation). When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.", + "description": "Control the user interface. Actions: toggle (turn tools on/off), open_panel (open a modal: documents/library, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), open_email_reply (open an email reply draft document; DOES NOT send. For 'write/draft a reply saying X', include body with the drafted reply), set_mode, switch_model, set_theme (built-in presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute), create_theme (CREATE any custom theme with a name + colors object — pick distinctive, evocative hex colors that match the requested aesthetic, NOT generic defaults. The theme auto-applies after creation). When a user asks for ANY theme not in the built-in preset list, ALWAYS use create_theme.", "parameters": { "type": "object", "properties": { @@ -426,6 +426,7 @@ FUNCTION_TOOL_SCHEMAS = [ "uid": {"type": "string", "description": "Email UID for open_email_reply"}, "folder": {"type": "string", "description": "Email folder for open_email_reply (default INBOX)"}, "mode": {"type": "string", "description": "Reply draft mode for open_email_reply: reply, reply-all, or ai-reply"}, + "body": {"type": "string", "description": "For open_email_reply: reply body to pre-fill. Required whenever the user told you what the reply should say. Opens a draft, does not send."}, "colors": {"type": "object", "description": "For create_theme: the theme colors", "properties": { "bg": {"type": "string", "description": "Background color (hex, e.g. #1a1a2e)"}, @@ -569,12 +570,12 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "manage_notes", - "description": "Manage notes and checklists (Google Keep-style): list, add, update, delete, toggle_item. IMPORTANT: For to-do lists / checklists, set note_type='checklist' and pass the items as the `checklist_items` array — do NOT serialize them into `content` as plain text. For freeform notes, use note_type='note' and put the body in `content`. `due_date` accepts natural language like 'tomorrow at 9am' (parsed in the user's timezone) and fires a notification — do not also create a calendar event for the same reminder.", + "description": "Manage notes and checklists (Google Keep-style): list, view, add, update, delete, toggle_item. Use list/search to find candidate notes, then view with the note id when you need the full body. IMPORTANT: For to-do lists / checklists, set note_type='checklist' and pass the items as the `checklist_items` array — do NOT serialize them into `content` as plain text. For freeform notes, use note_type='note' and put the body in `content`. `due_date` accepts natural language like 'tomorrow at 9am' (parsed in the user's timezone) and fires a notification — do not also create a calendar event for the same reminder.", "parameters": { "type": "object", "properties": { "action": {"type": "string", - "enum": ["list", "add", "update", "delete", "toggle_item"], + "enum": ["list", "view", "add", "update", "delete", "toggle_item"], "description": "The action to perform"}, "id": {"type": "string", "description": "Note id (for update/delete/toggle_item); 8-char prefix is fine"}, "title": {"type": "string", "description": "Note title (for add/update)"}, @@ -1106,7 +1107,7 @@ FUNCTION_TOOL_SCHEMAS = [ "type": "function", "function": { "name": "reply_to_email", - "description": "SEND a reply email immediately by UID. Do not use this when the user asks to open/start a reply window or draft; use ui_control action=open_email_reply instead. For follow-up 'reply ...' requests where the user clearly wants to send now, use the exact UID from the latest read_email/list_emails result; never invent UID 1. Automatically threads with In-Reply-To/References headers.", + "description": "SEND a reply email immediately by UID. Do not use this when the user asks to write/draft/open/start a reply; use ui_control action=open_email_reply with body instead so the user can review. Only use when the user explicitly says to send now. Use the exact UID from the latest read_email/list_emails result; never invent UID 1. Automatically threads with In-Reply-To/References headers.", "parameters": { "type": "object", "properties": { @@ -1373,6 +1374,9 @@ def function_call_to_tool_block(name: str, arguments: str) -> Optional[ToolBlock folder = args.get("folder") or value or "INBOX" mode = args.get("mode") or "reply" content = f"open_email_reply {uid} {folder} {mode}" + body = args.get("body") or args.get("extra") or args.get("content") or "" + if body: + content += f" {body}" elif action == "set_mode": content = f"set_mode {value or name}" elif action == "switch_model": diff --git a/static/app.js b/static/app.js index 1f0390a37..d1a8c9f1a 100644 --- a/static/app.js +++ b/static/app.js @@ -1617,6 +1617,7 @@ function initializeEventListeners() { // Delay tool glow-up for a staggered effect setTimeout(() => applyModeToToggles(mode), 500); } + window.__odysseusSetChatMode = setMode; agentBtn.addEventListener('click', () => { // Agent mode turns off research if active const resChk = el('research-toggle'); @@ -1692,10 +1693,29 @@ function initializeEventListeners() { try { workspaceModule.initWorkspace(); } catch (_) {} // Document editor toggle (special: uses module panel, not a checkbox) + function bringOpenDocumentToFrontOnMobile() { + if (window.innerWidth > 768) return false; + if (!documentModule || !documentModule.isPanelOpen || !documentModule.isPanelOpen()) return false; + if (!document.body.classList.contains('email-front')) return false; + document.body.classList.remove('email-front', 'email-doc-split-active'); + document.documentElement.style.removeProperty('--email-doc-split-left-x'); + document.documentElement.style.removeProperty('--email-doc-split-email-w'); + document.documentElement.style.removeProperty('--email-doc-split-right-x'); + const docPane = document.getElementById('doc-editor-pane'); + if (docPane) docPane.style.setProperty('z-index', '10010', 'important'); + const overflow = el('overflow-doc-btn'); + if (overflow) overflow.classList.add('active'); + const indicator = el('doc-indicator-btn'); + if (indicator) indicator.classList.add('active'); + const st = loadToggleState(); st.doc = true; saveToggleState(st); + return true; + } + const overflowDocBtn = el('overflow-doc-btn'); if (overflowDocBtn) { overflowDocBtn.addEventListener('click', async () => { if (!documentModule) return; + if (bringOpenDocumentToFrontOnMobile()) return; if (documentModule.isPanelOpen()) { documentModule.closePanel(); overflowDocBtn.classList.remove('active'); @@ -2304,14 +2324,32 @@ function initializeEventListeners() { // IMPORTANT: don't overwrite the user's persisted per-mode tool prefs // (`web_agent`, `bash_agent`, `web_chat`, `bash_chat`). Nobody mode is // ephemeral — their agent-mode defaults must come back on toggle-off. + const beforeNobody = Storage.getJSON(Storage.KEYS.TOGGLES, {}) || {}; + if (!beforeNobody.nobody_prev_mode) beforeNobody.nobody_prev_mode = beforeNobody.mode || 'agent'; + Storage.setJSON(Storage.KEYS.TOGGLES, beforeNobody); const _offIds = ['web-toggle', 'bash-toggle', 'research-toggle']; _offIds.forEach(id => { const c = el(id); if (c) c.checked = false; }); ['web-toggle-btn', 'bash-toggle-btn'].forEach(id => { const b = el(id); if (b) b.classList.remove('active'); }); - const _ab = el('mode-agent-btn'), _cb = el('mode-chat-btn'); - if (_ab) _ab.classList.remove('active'); - if (_cb) _cb.classList.add('active'); + if (typeof window.__odysseusSetChatMode === 'function') { + window.__odysseusSetChatMode('chat'); + } else { + const _ab = el('mode-agent-btn'), _cb = el('mode-chat-btn'); + if (_ab) { + _ab.classList.remove('active'); + _ab.setAttribute('aria-pressed', 'false'); + } + if (_cb) { + _cb.classList.add('active'); + _cb.setAttribute('aria-pressed', 'true'); + } + const _toggle = _ab?.closest('.mode-toggle') || _cb?.closest('.mode-toggle'); + if (_toggle) _toggle.classList.add('mode-chat'); + const ts = Storage.getJSON(Storage.KEYS.TOGGLES, {}); + ts.mode = 'chat'; + Storage.setJSON(Storage.KEYS.TOGGLES, ts); + } const ts = Storage.getJSON(Storage.KEYS.TOGGLES, {}); - ts.research = false; ts.mode = 'chat'; + ts.research = false; Storage.setJSON(Storage.KEYS.TOGGLES, ts); } else { incognitoBtn.innerHTML = INCOGNITO_EYE_OPEN + 'Nobody'; @@ -2335,11 +2373,15 @@ function initializeEventListeners() { // Heal any previously-persisted false values from the old Nobody bug // so agent-mode defaults (web/bash ON) come back. const _ts = Storage.getJSON(Storage.KEYS.TOGGLES, {}); - let _dirty = false; + const _restoreMode = _ts.nobody_prev_mode || 'agent'; + delete _ts.nobody_prev_mode; ['web_agent', 'bash_agent', 'web_chat', 'bash_chat'].forEach(k => { - if (_ts[k] === false) { delete _ts[k]; _dirty = true; } + if (_ts[k] === false) delete _ts[k]; }); - if (_dirty) Storage.setJSON(Storage.KEYS.TOGGLES, _ts); + Storage.setJSON(Storage.KEYS.TOGGLES, _ts); + if (typeof window.__odysseusSetChatMode === 'function') { + window.__odysseusSetChatMode(_restoreMode === 'chat' ? 'chat' : 'agent'); + } // Reapply the current mode's real defaults to the visible toggles const _curMode = (Storage.getJSON(Storage.KEYS.TOGGLES, {}) || {}).mode || 'chat'; try { applyModeToToggles(_curMode); } catch (_) {} diff --git a/static/index.html b/static/index.html index b20b73428..32386b988 100644 --- a/static/index.html +++ b/static/index.html @@ -1952,6 +1952,34 @@ +
+

Translation

+
+
+
Auto translate
+
When an opened email appears to be in another language, prepare a translated view.
+
+ +
+
+ + + + + + + + + + + + + + + +
+
+

Writing Style

AI-extracted from your sent emails. Used when AI drafts replies.
diff --git a/static/js/chat.js b/static/js/chat.js index c0b91f980..9bc582b41 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -821,11 +821,14 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer // Web toggle: pre-search in Chat mode, tool permission in Agent mode const toggleState = Storage.loadToggleState(); let isAgentMode = (toggleState.mode || 'chat') === 'agent'; + const incognitoChk = el('incognito-toggle'); + const isIncognito = !!(incognitoChk && incognitoChk.checked); // Auto-escalate to agent mode when a document is open — the user expects // the AI to see the document and have tools to edit it - if (!isAgentMode && documentModule && documentModule.isPanelOpen() && documentModule.getCurrentDocId()) { + if (!isIncognito && !isAgentMode && documentModule && documentModule.isPanelOpen() && documentModule.getCurrentDocId()) { isAgentMode = true; } + if (isIncognito) isAgentMode = false; fd.append('mode', isAgentMode ? 'agent' : 'chat'); if (el('web-toggle').checked) { if (isAgentMode) { @@ -846,8 +849,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer if (ragChk && !ragChk.checked) { fd.append('use_rag', 'false'); } - const incognitoChk = el('incognito-toggle'); - if (incognitoChk && incognitoChk.checked) { + if (isIncognito) { fd.append('incognito', 'true'); } const _ws = (Storage.KEYS && Storage.get(Storage.KEYS.WORKSPACE, '')) || ''; @@ -864,7 +866,7 @@ import { wireArrowUpRecall, getLastUserMessageFromChatHistory } from './composer currentAbort = abortCtrl; const _tState = Storage.loadToggleState(); - const _isAgent = (_tState.mode || 'chat') === 'agent'; + const _isAgent = !isIncognito && (_tState.mode || 'chat') === 'agent'; // Timeout: 6 min for research and agent mode, 3 min otherwise const timeoutMs = el('research-toggle').checked || _isAgent ? RESEARCH_TIMEOUT_MS : DEFAULT_TIMEOUT_MS; diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index 253fa5724..d0d9d6f45 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -1068,6 +1068,17 @@ document.addEventListener('click', function(e) { } }, true); +function resolveDocumentPlaceholderLinks(text, metadata) { + if (!text || !metadata || !Array.isArray(metadata.tool_events)) return text; + const docEvents = metadata.tool_events.filter(ev => ev && ev.doc_id); + if (!docEvents.length) return text; + return String(text).replace(/#document-(\d+)\b/g, (match, num) => { + const idx = Number(num) - 1; + const ev = Number.isInteger(idx) && idx >= 0 ? docEvents[idx] : null; + return ev && ev.doc_id ? `#document-${ev.doc_id}` : match; + }); +} + // Jump-to-entity anchors — the agent emits links like // [New Chat](#session-89effa28) // [Notes](#document-abc123) @@ -2005,7 +2016,7 @@ export function addMessage(role, content, modelName, metadata) { for (let r = 0; r < maxRound; r++) { const roundNum = r + 1; - const txt = (roundTexts[r] || '').trim(); + const txt = resolveDocumentPlaceholderLinks((roundTexts[r] || '').trim(), metadata); if (txt) { const wrap = document.createElement('div'); @@ -2186,6 +2197,9 @@ export function addMessage(role, content, modelName, metadata) { b.className = 'body'; let text = markdownModule.squashOutsideCode(stripToolBlocks(textRaw || '')); + if (role === 'assistant') { + text = resolveDocumentPlaceholderLinks(text, metadata); + } // For user messages, pull out vision-model image descriptions ([Image: name]\n // ) into a collapsible "image description" section. Done for diff --git a/static/js/cookbook-hwfit.js b/static/js/cookbook-hwfit.js index ed2b29940..9e5cda02d 100644 --- a/static/js/cookbook-hwfit.js +++ b/static/js/cookbook-hwfit.js @@ -577,6 +577,7 @@ export async function _hwfitFetch(fresh = false) { const _sig = _scanSig(); const _cached = fresh ? null : _readScanCache(_sig); const wp = spinnerModule.createWhirlpool(18); + const _paintedFromCache = !!_cached; if (_cached) { // Tag the restored cache with its host too (scan-sig keys cache per // host, so a hit here is always for the current remoteHost). @@ -635,6 +636,10 @@ export async function _hwfitFetch(fresh = false) { }); }).catch(() => {}); } + if (_paintedFromCache) { + try { wp.destroy(); } catch {} + return; + } try { const sortBy = document.getElementById('hwfit-sort')?.value || 'newest'; const quantPref = document.getElementById('hwfit-quant')?.value || ''; @@ -1958,16 +1963,17 @@ export function _hwfitInit() { dot.className = 'cookbook-srv-status testing'; dot.title = 'Testing SSH…'; setMsg('Testing SSH...'); - const pf = port && port !== '22' ? `-p ${port} ` : ''; - const cmd = `ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new ${pf}${host} "echo ok"`; const t0 = Date.now(); try { - const res = await fetch('/api/shell/exec', { + const res = await fetch('/api/cookbook/test-ssh', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: cmd, timeout: 8 }), + body: JSON.stringify({ host, ssh_port: port || undefined }), }); const data = await res.json(); + if (!res.ok) { + throw new Error(data.detail || data.error || `HTTP ${res.status}`); + } const ms = Date.now() - t0; const out = (data.stdout || '').trim(); if (data.exit_code === 0 && out.startsWith('ok')) { @@ -1976,7 +1982,7 @@ export function _hwfitInit() { setMsg(`Connected · ${ms} ms`, 'var(--green,#50fa7b)'); } else { dot.className = 'cookbook-srv-status fail'; - const err = (data.stderr || data.stdout || `exit ${data.exit_code}`).toString().trim().slice(0, 240); + const err = (data.stderr || data.stdout || (data.exit_code == null ? 'no exit code' : `exit ${data.exit_code}`)).toString().trim().slice(0, 240); dot.title = `SSH failed: ${err}`; setMsg(`Failed · ${err}`, 'var(--red,#e06c75)'); } diff --git a/static/js/cookbook.js b/static/js/cookbook.js index 43a3ad5d0..3206948c2 100644 --- a/static/js/cookbook.js +++ b/static/js/cookbook.js @@ -873,8 +873,9 @@ async function _fetchDependencies() { let _spin = null; try { const sp = (await import('./spinner.js')).default; - _spin = sp.createWhirlpool(28); - _spin.element.style.cssText = 'margin:24px auto 0;display:block;'; + _spin = sp.createWhirlpool(22); + _spin.element.classList.add('cookbook-section-loading-wp'); + _spin.element.style.cssText = 'margin:24px auto 0;display:block;width:22px;height:22px;'; list.appendChild(_spin.element); const label = document.createElement('div'); label.className = 'hwfit-loading'; @@ -1788,7 +1789,7 @@ function _wireTabEvents(body) { const scanBtn = document.getElementById('hwfit-cache-scan'); if (scanBtn) { - scanBtn.addEventListener('click', () => _fetchCachedModels()); + scanBtn.addEventListener('click', () => _fetchCachedModels(true)); } const editDirsLink = document.querySelector('.cookbook-serve-dir-edit'); @@ -2286,8 +2287,9 @@ function _wireTabEvents(body) { hfList.innerHTML = ''; try { const sp = (await import('./spinner.js')).default; - const _spin = sp.createWhirlpool(28); - _spin.element.style.cssText = 'margin:24px auto 0;display:block;'; + const _spin = sp.createWhirlpool(22); + _spin.element.classList.add('cookbook-section-loading-wp'); + _spin.element.style.cssText = 'margin:24px auto 0;display:block;width:22px;height:22px;'; hfList.appendChild(_spin.element); const lbl = document.createElement('div'); lbl.className = 'hwfit-loading'; diff --git a/static/js/cookbookServe.js b/static/js/cookbookServe.js index 06a990b82..0e168da60 100644 --- a/static/js/cookbookServe.js +++ b/static/js/cookbookServe.js @@ -45,6 +45,30 @@ const SERVE_STATE_KEY = 'cookbook-serve-state'; const SERVE_FAVORITES_KEY = 'cookbook-serve-favorite-models'; let _cachedAllModels = []; +const _CACHED_MODELS_SCAN_KEY = 'cookbook_cached_models_scan_v1'; +const _CACHED_MODELS_SCAN_TTL = 6 * 3600 * 1000; + +function _readCachedModelScan(sig) { + try { + const all = JSON.parse(localStorage.getItem(_CACHED_MODELS_SCAN_KEY) || '{}'); + const entry = all[sig]; + if (entry && Date.now() - (entry.ts || 0) < _CACHED_MODELS_SCAN_TTL) return entry.data || null; + } catch {} + return null; +} + +function _writeCachedModelScan(sig, data) { + try { + const all = JSON.parse(localStorage.getItem(_CACHED_MODELS_SCAN_KEY) || '{}'); + all[sig] = { ts: Date.now(), data }; + const keys = Object.keys(all); + if (keys.length > 12) { + keys.sort((a, b) => (all[a].ts || 0) - (all[b].ts || 0)); + for (const k of keys.slice(0, keys.length - 12)) delete all[k]; + } + localStorage.setItem(_CACHED_MODELS_SCAN_KEY, JSON.stringify(all)); + } catch {} +} function _loadServeFavorites() { try { @@ -3563,12 +3587,91 @@ export async function openServePanelForRepo(repo, fields) { // ── Fetch cached models from server ── -export async function _fetchCachedModels() { +function _renderCachedModelsData(list, data, host) { + // CHANGELOG: 'ready' already excludes partial downloads; + // show every complete model regardless of size/backend. + const ready = (data.models || []).filter(m => m.status === 'ready'); + + const downloading = (data.models || []).filter(m => m.status === 'downloading'); + const allModels = [...ready, ...downloading]; + _cachedAllModels = allModels; + + if (!allModels.length) { + if (!host) { + list.innerHTML = '
No cached models found
Docker Local uses Odysseus’s cache in data/huggingface. Download a model here, or copy an existing host HuggingFace cache into that folder once.
'; + } else { + list.innerHTML = '
No cached models found
'; + } + const tagContainer = document.getElementById('serve-tags'); + if (tagContainer) tagContainer.innerHTML = ''; + return; + } + + // Auto-detect type + family tags + const _tagMap = {}; + const _familyMap = {}; + const _families = [ + [/qwen/i, 'qwen'], [/llama/i, 'llama'], [/mistral|mixtral/i, 'mistral'], + [/deepseek/i, 'deepseek'], [/gemma/i, 'gemma'], [/phi/i, 'phi'], + [/minimax/i, 'minimax'], [/glm/i, 'glm'], [/flux/i, 'flux'], + [/stable.?diffusion|sdxl/i, 'sd'], [/z-image/i, 'z-image'], + [/whisper/i, 'whisper'], [/command|cohere/i, 'cohere'], + [/yi-/i, 'yi'], [/intern/i, 'intern'], [/falcon/i, 'falcon'], + ]; + for (const m of allModels) { + const n = (m.repo_id || '').toLowerCase(); + let tag = 'other'; + if (m.backend === 'ollama' || m.is_ollama) tag = 'llm'; + else if (m.is_diffusion || /flux|sdxl|stable-diffusion|z-image|qwen-image|diffusion|dreamshar/i.test(n)) tag = 'image'; + else if (/whisper|stt|asr/i.test(n)) tag = 'stt'; + else if (/tts|cosyvoice|parler/i.test(n)) tag = 'tts'; + else if (/embed|bge|minilm|e5-/i.test(n)) tag = 'embedding'; + else if (/lora|adapter/i.test(n)) tag = 'lora'; + else tag = 'llm'; + m._tag = tag; + _tagMap[tag] = (_tagMap[tag] || 0) + 1; + m._family = ''; + for (const [re, fam] of _families) { + if (re.test(n)) { m._family = fam; _familyMap[fam] = (_familyMap[fam] || 0) + 1; break; } + } + if ((m.backend === 'ollama' || m.is_ollama) && !m._family) { + m._family = 'ollama'; + _familyMap.ollama = (_familyMap.ollama || 0) + 1; + } + } + + // Render tag chips + const tagContainer = document.getElementById('serve-tags'); + if (tagContainer) { + const tagOrder = ['llm', 'image', 'lora', 'embedding', 'tts', 'stt', 'other']; + let tagHtml = ``; + for (const t of tagOrder) { + if (!_tagMap[t]) continue; + tagHtml += ``; + } + const sortedFamilies = Object.entries(_familyMap).sort((a, b) => b[1] - a[1]); + if (sortedFamilies.length) { + for (const [fam, count] of sortedFamilies) { + const logo = providerLogo(fam); + const logoHtml = logo ? `${logo}` : ''; + tagHtml += ``; + } + } + tagContainer.innerHTML = tagHtml; + } + + _rerenderCachedModels(); +} + +export async function _fetchCachedModels(fresh = false) { const list = document.getElementById('hwfit-cached-list'); if (!list) return; list.innerHTML = ''; - const _dlWp = spinnerModule.createWhirlpool(18); + const _dlWp = spinnerModule.createWhirlpool(22); + _dlWp.element.classList.add('cookbook-section-loading-wp'); + _dlWp.element.style.width = '22px'; + _dlWp.element.style.height = '22px'; const _dlWrap = document.createElement('div'); _dlWrap.className = 'hwfit-loading'; _dlWrap.style.cssText = 'flex-direction:column;gap:6px;'; @@ -3630,6 +3733,13 @@ export async function _fetchCachedModels() { if (host) { qp.set('host', host); const _sp4 = _getPort(host); if (_sp4) qp.set('ssh_port', _sp4); const _plat = _getPlatform(host); if (_plat) qp.set('platform', _plat); } if (modelDirs.length) qp.set('model_dir', modelDirs.join(',')); const params = qp.toString() ? `?${qp}` : ''; + const scanSig = params || 'local'; + const cached = fresh ? null : _readCachedModelScan(scanSig); + if (cached) { + _dlWp.destroy(); + _renderCachedModelsData(list, cached, host); + return; + } const res = await fetch(`/api/model/cached${params}`); if (!res.ok) { const body = await res.text().catch(() => ''); @@ -3644,80 +3754,9 @@ export async function _fetchCachedModels() { throw new Error(`HTTP ${res.status} ${res.statusText}${msg ? `: ${msg}` : ''}`); } const data = await res.json(); + _writeCachedModelScan(scanSig, data); _dlWp.destroy(); - - // CHANGELOG: 'ready' already excludes partial downloads; - // show every complete model regardless of size/backend. - const ready = data.models.filter(m => m.status === 'ready'); - - const downloading = data.models.filter(m => m.status === 'downloading'); - const allModels = [...ready, ...downloading]; - _cachedAllModels = allModels; - - if (!allModels.length) { - if (!host) { - list.innerHTML = '
No cached models found
Docker Local uses Odysseus’s cache in data/huggingface. Download a model here, or copy an existing host HuggingFace cache into that folder once.
'; - } else { - list.innerHTML = '
No cached models found
'; - } - document.getElementById('serve-tags').innerHTML = ''; - return; - } - - // Auto-detect type + family tags - const _tagMap = {}; - const _familyMap = {}; - const _families = [ - [/qwen/i, 'qwen'], [/llama/i, 'llama'], [/mistral|mixtral/i, 'mistral'], - [/deepseek/i, 'deepseek'], [/gemma/i, 'gemma'], [/phi/i, 'phi'], - [/minimax/i, 'minimax'], [/glm/i, 'glm'], [/flux/i, 'flux'], - [/stable.?diffusion|sdxl/i, 'sd'], [/z-image/i, 'z-image'], - [/whisper/i, 'whisper'], [/command|cohere/i, 'cohere'], - [/yi-/i, 'yi'], [/intern/i, 'intern'], [/falcon/i, 'falcon'], - ]; - for (const m of allModels) { - const n = (m.repo_id || '').toLowerCase(); - let tag = 'other'; - if (m.backend === 'ollama' || m.is_ollama) tag = 'llm'; - else if (m.is_diffusion || /flux|sdxl|stable-diffusion|z-image|qwen-image|diffusion|dreamshar/i.test(n)) tag = 'image'; - else if (/whisper|stt|asr/i.test(n)) tag = 'stt'; - else if (/tts|cosyvoice|parler/i.test(n)) tag = 'tts'; - else if (/embed|bge|minilm|e5-/i.test(n)) tag = 'embedding'; - else if (/lora|adapter/i.test(n)) tag = 'lora'; - else tag = 'llm'; - m._tag = tag; - _tagMap[tag] = (_tagMap[tag] || 0) + 1; - m._family = ''; - for (const [re, fam] of _families) { - if (re.test(n)) { m._family = fam; _familyMap[fam] = (_familyMap[fam] || 0) + 1; break; } - } - if ((m.backend === 'ollama' || m.is_ollama) && !m._family) { - m._family = 'ollama'; - _familyMap.ollama = (_familyMap.ollama || 0) + 1; - } - } - - // Render tag chips - const tagContainer = document.getElementById('serve-tags'); - if (tagContainer) { - const tagOrder = ['llm', 'image', 'lora', 'embedding', 'tts', 'stt', 'other']; - let tagHtml = ``; - for (const t of tagOrder) { - if (!_tagMap[t]) continue; - tagHtml += ``; - } - const sortedFamilies = Object.entries(_familyMap).sort((a, b) => b[1] - a[1]); - if (sortedFamilies.length) { - for (const [fam, count] of sortedFamilies) { - const logo = providerLogo(fam); - const logoHtml = logo ? `${logo}` : ''; - tagHtml += ``; - } - } - tagContainer.innerHTML = tagHtml; - } - - _rerenderCachedModels(); + _renderCachedModelsData(list, data, host); } catch (e) { _dlWp.destroy(); list.innerHTML = `
Failed: ${esc(e.message)}
`; diff --git a/static/js/document.js b/static/js/document.js index 3fb825656..d2c3584ec 100644 --- a/static/js/document.js +++ b/static/js/document.js @@ -31,6 +31,11 @@ import * as Modals from './modalManager.js'; let _emailAccountsCache = null; let _emailAccountsCacheAt = 0; let _emailHeaderManualExpandUntil = 0; + let _emailStreamAnimFrame = null; + let _emailStreamRenderedBody = ''; + let _emailStreamTargetBody = ''; + let _emailLocalDraftDebounce = null; + const _EMAIL_LOCAL_DRAFT_PREFIX = 'odysseus.email.replyDraft.v1:'; // Diff mode state let _diffModeActive = false; @@ -38,6 +43,8 @@ import * as Modals from './modalManager.js'; let _diffNewContent = null; let _diffChunks = []; // [{id, oldLines, newLines, startLine, resolved, accepted}] let _diffUnresolvedCount = 0; + let _mdPreviewClickTimes = []; + let _mdPreviewHintLastAt = 0; // Language auto-detection config const AUTO_DETECT_DELAY = 500; @@ -2045,9 +2052,8 @@ import * as Modals from './modalManager.js'; || ''; const isForm = _isFormBackedDoc(live); // Footer main button: for a doc opened from an email attachment, morph the - // Copy button into "Reply" (send the filled file back to the sender via the - // signed-reply flow). Otherwise it's the normal Copy action. The click - // handler branches on data-mode. + // Save button into "Attach" (send the filled file back to the sender via + // the signed-reply flow). Otherwise it forces a new saved version. const _copyBtn = document.getElementById('doc-footer-copy-btn'); if (_copyBtn) { const _ad = docs.get(activeDocId); @@ -2056,10 +2062,10 @@ import * as Modals from './modalManager.js'; _copyBtn.dataset.mode = 'reply'; _copyBtn.title = 'Reply to the sender with this filled file attached'; _copyBtn.innerHTML = 'Attach'; - } else if (!_replyable && _copyBtn.dataset.mode !== 'copy') { - _copyBtn.dataset.mode = 'copy'; - _copyBtn.title = 'Copy document'; - _copyBtn.innerHTML = 'Copy'; + } else if (!_replyable && _copyBtn.dataset.mode !== 'save') { + _copyBtn.dataset.mode = 'save'; + _copyBtn.title = 'Save new version'; + _copyBtn.innerHTML = 'Save'; } } // Standalone Export PDF / PDF-toggle icon buttons are retired — for a @@ -2180,6 +2186,7 @@ import * as Modals from './modalManager.js'; if (mdToggle) { mdToggle.querySelector('[data-mdview="edit"]')?.classList.toggle('active', !_mdActive); mdToggle.querySelector('[data-mdview="preview"]')?.classList.toggle('active', _mdActive); + mdToggle.classList.toggle('is-preview-active', !!_mdActive); } } else if (lang === 'csv') { show = true; @@ -2206,6 +2213,16 @@ import * as Modals from './modalManager.js'; // suppress the single morph button to avoid two redundant controls. if (_hasViewToggle(lang)) show = false; actionBtn.style.display = show ? '' : 'none'; + document.querySelectorAll('.md-toolbar-edit-only').forEach(el => { + el.style.display = (lang === 'markdown' && _mdActive) ? 'none' : ''; + }); + const mdToolbar = document.getElementById('doc-md-toolbar'); + if (mdToolbar) { + mdToolbar.classList.toggle('md-preview-active', lang === 'markdown' && !!_mdActive); + mdToolbar.classList.toggle('md-write-active', lang === 'markdown' && !_mdActive); + } + if (_mdPreview) _mdPreview.classList.toggle('md-preview-active', lang === 'markdown' && !!_mdActive); + if (mdToolbar && mdToolbar._syncOverflow) requestAnimationFrame(mdToolbar._syncOverflow); // Now that the contextual buttons' visibility is settled, collapse the bar // if it ended up empty (the common plain-doc-on-mobile case). @@ -2215,20 +2232,24 @@ import * as Modals from './modalManager.js'; // ── Email document type helpers ── function _parseEmailHeader(content) { - const empty = { to: '', cc: '', bcc: '', subject: '', inReplyTo: '', references: '', sourceUid: '', sourceFolder: '', attachments: [], body: content || '' }; + const empty = { to: '', cc: '', bcc: '', subject: '', inReplyTo: '', references: '', sourceUid: '', sourceFolder: '', forwardAttachments: false, attachments: [], body: content || '' }; if (!content) return empty; const parts = content.split(/\n---\n/); if (parts.length < 2) return empty; const header = parts[0]; const body = parts.slice(1).join('\n---\n'); - const fields = { to: '', cc: '', bcc: '', subject: '', inReplyTo: '', references: '', sourceUid: '', sourceFolder: '', attachments: [], body: body }; + const fields = { to: '', cc: '', bcc: '', subject: '', inReplyTo: '', references: '', sourceUid: '', sourceFolder: '', forwardAttachments: false, attachments: [], body: body }; for (const line of header.split('\n')) { - const m = line.match(/^(To|Cc|Bcc|Subject|In-Reply-To|References|X-Source-UID|X-Source-Folder|X-Attachments):\s*(.*)$/i); + const m = line.match(/^(To|Cc|Bcc|Subject|In-Reply-To|References|X-Source-UID|X-Source-Folder|X-Forward-Attachments|X-Attachments):\s*(.*)$/i); if (m) { let key = m[1].toLowerCase(); if (key === 'in-reply-to') key = 'inReplyTo'; else if (key === 'x-source-uid') key = 'sourceUid'; else if (key === 'x-source-folder') key = 'sourceFolder'; + else if (key === 'x-forward-attachments') { + fields.forwardAttachments = /^(1|true|yes)$/i.test((m[2] || '').trim()); + continue; + } else if (key === 'x-attachments') { fields.attachments = m[2].trim().split('|').map(a => { const [index, filename, size] = a.split(':'); @@ -2254,6 +2275,95 @@ import * as Modals from './modalManager.js'; return header + '\n---\n' + body; } + function _emailLocalDraftKey(sourceUid, sourceFolder, inReplyTo) { + const uid = String(sourceUid || '').trim(); + if (!uid) return ''; + const folder = String(sourceFolder || 'INBOX').trim() || 'INBOX'; + const msg = String(inReplyTo || '').trim(); + return _EMAIL_LOCAL_DRAFT_PREFIX + encodeURIComponent(`${folder}|${uid}|${msg}`); + } + + function _loadEmailLocalDraft(fields) { + const key = _emailLocalDraftKey(fields?.sourceUid, fields?.sourceFolder, fields?.inReplyTo); + if (!key) return null; + try { + const raw = localStorage.getItem(key); + if (!raw) return null; + const draft = JSON.parse(raw); + if (!draft || typeof draft !== 'object') return null; + const updatedAt = Number(draft.updatedAt || 0); + if (updatedAt && Date.now() - updatedAt > 45 * 24 * 60 * 60 * 1000) { + localStorage.removeItem(key); + return null; + } + return draft; + } catch (_) { + return null; + } + } + + function _emailFieldsWithLocalDraft(fields) { + const draft = _loadEmailLocalDraft(fields); + if (!draft) return fields; + return { + ...fields, + to: draft.to ?? fields.to, + cc: draft.cc ?? fields.cc, + bcc: draft.bcc ?? fields.bcc, + subject: draft.subject ?? fields.subject, + inReplyTo: draft.inReplyTo ?? fields.inReplyTo, + references: draft.references ?? fields.references, + sourceUid: draft.sourceUid ?? fields.sourceUid, + sourceFolder: draft.sourceFolder ?? fields.sourceFolder, + body: draft.body ?? fields.body, + }; + } + + function _persistEmailLocalDraftNow() { + const doc = activeDocId && docs.get(activeDocId); + if (!doc || doc.language !== 'email') return; + const sourceUid = document.getElementById('doc-email-source-uid')?.value || ''; + const sourceFolder = document.getElementById('doc-email-source-folder')?.value || 'INBOX'; + const inReplyTo = document.getElementById('doc-email-in-reply-to')?.value || ''; + const key = _emailLocalDraftKey(sourceUid, sourceFolder, inReplyTo); + if (!key) return; + const rich = document.getElementById('doc-email-richbody'); + const textarea = document.getElementById('doc-editor-textarea'); + const body = (rich && rich.style.display !== 'none') ? rich.innerHTML : (textarea?.value || ''); + const payload = { + to: document.getElementById('doc-email-to')?.value || '', + cc: document.getElementById('doc-email-cc')?.value || '', + bcc: document.getElementById('doc-email-bcc')?.value || '', + subject: document.getElementById('doc-email-subject')?.value || '', + inReplyTo, + references: document.getElementById('doc-email-references')?.value || '', + sourceUid, + sourceFolder, + body, + updatedAt: Date.now(), + }; + try { localStorage.setItem(key, JSON.stringify(payload)); } catch (_) {} + } + + function _persistEmailLocalDraftSoon() { + clearTimeout(_emailLocalDraftDebounce); + _emailLocalDraftDebounce = setTimeout(_persistEmailLocalDraftNow, 250); + } + + function _clearEmailLocalDraft(sourceUid, sourceFolder, inReplyTo) { + const key = _emailLocalDraftKey(sourceUid, sourceFolder, inReplyTo); + if (!key) return; + try { localStorage.removeItem(key); } catch (_) {} + } + + function _clearCurrentEmailLocalDraft() { + _clearEmailLocalDraft( + document.getElementById('doc-email-source-uid')?.value || '', + document.getElementById('doc-email-source-folder')?.value || 'INBOX', + document.getElementById('doc-email-in-reply-to')?.value || '', + ); + } + // ── WYSIWYG email body helpers ── function _emailBodyToHtml(text) { const t = (text || '').trim(); @@ -2282,7 +2392,10 @@ import * as Modals from './modalManager.js'; function _wireEmailRichbody(rich) { if (rich._wired) { _syncEmailRichbody(rich); return; } rich._wired = true; - rich.addEventListener('input', () => _syncEmailRichbody(rich)); + rich.addEventListener('input', () => { + _syncEmailRichbody(rich); + _persistEmailLocalDraftSoon(); + }); // Highlight toolbar buttons (B / I / S, headings, lists) when the caret // sits inside formatted text. queryCommandState reflects the live // selection — we just translate that into .is-active classes the CSS @@ -2374,6 +2487,53 @@ import * as Modals from './modalManager.js'; }); } + function _renderStreamingEmailBody(body, { immediate = false } = {}) { + const rich = document.getElementById('doc-email-richbody'); + const textarea = document.getElementById('doc-editor-textarea'); + if (!rich) return; + + _emailStreamTargetBody = body || ''; + if (!_emailStreamRenderedBody && textarea && textarea.value) { + _emailStreamRenderedBody = textarea.value; + } + + const applyBody = (value) => { + if (textarea) { + textarea.value = value; + textarea.scrollTop = textarea.scrollHeight; + } + rich.innerHTML = _emailBodyToHtml(value); + rich.scrollTop = rich.scrollHeight; + }; + + if (immediate) { + if (_emailStreamAnimFrame) cancelAnimationFrame(_emailStreamAnimFrame); + _emailStreamAnimFrame = null; + _emailStreamRenderedBody = _emailStreamTargetBody; + applyBody(_emailStreamRenderedBody); + return; + } + + if (_emailStreamTargetBody.length < _emailStreamRenderedBody.length || + !_emailStreamTargetBody.startsWith(_emailStreamRenderedBody)) { + _emailStreamRenderedBody = ''; + } + + if (_emailStreamAnimFrame) return; + const tick = () => { + const remaining = _emailStreamTargetBody.length - _emailStreamRenderedBody.length; + if (remaining <= 0) { + _emailStreamAnimFrame = null; + return; + } + const step = Math.max(1, Math.min(8, Math.ceil(remaining / 18))); + _emailStreamRenderedBody = _emailStreamTargetBody.slice(0, _emailStreamRenderedBody.length + step); + applyBody(_emailStreamRenderedBody); + _emailStreamAnimFrame = requestAnimationFrame(tick); + }; + _emailStreamAnimFrame = requestAnimationFrame(tick); + } + function _stripEmailReplyQuoteText(text) { const original = String(text || ''); if (!original) return { body: '', stripped: false }; @@ -2397,6 +2557,7 @@ import * as Modals from './modalManager.js'; syncHighlighting(); const rich = _emailRichbodyActive(); if (rich) rich.innerHTML = _emailBodyToHtml(textarea.value); + _persistEmailLocalDraftSoon(); } async function _streamEmailBodyText(textarea, value) { @@ -2411,6 +2572,7 @@ import * as Modals from './modalManager.js'; const next = finalText.slice(0, i + chunk); textarea.value = next; if (rich) rich.innerHTML = _emailBodyToHtml(next); + _persistEmailLocalDraftSoon(); await new Promise(resolve => requestAnimationFrame(resolve)); } _setEmailBodyText(textarea, finalText); @@ -2509,7 +2671,8 @@ import * as Modals from './modalManager.js'; document.getElementById('doc-editor-textarea')?.classList.add('email-mode'); document.getElementById('doc-editor-code')?.classList.add('email-mode'); document.getElementById('doc-editor-highlight')?.classList.add('email-mode'); - const fields = _parseEmailHeader(doc.content || ''); + let fields = _parseEmailHeader(doc.content || ''); + fields = _emailFieldsWithLocalDraft(fields); const toInput = document.getElementById('doc-email-to'); const subjectInput = document.getElementById('doc-email-subject'); const inReplyTo = document.getElementById('doc-email-in-reply-to'); @@ -2635,6 +2798,10 @@ import * as Modals from './modalManager.js'; if (_rich && _srcWrap) { _srcWrap.style.display = 'none'; _rich.style.display = ''; + if (_emailStreamAnimFrame) cancelAnimationFrame(_emailStreamAnimFrame); + _emailStreamAnimFrame = null; + _emailStreamRenderedBody = fields.body || ''; + _emailStreamTargetBody = fields.body || ''; _rich.innerHTML = _emailBodyToHtml(fields.body); _wireEmailRichbody(_rich); setTimeout(() => { @@ -2660,6 +2827,49 @@ import * as Modals from './modalManager.js'; if (bccRow) bccRow.style.display = hasCcBcc ? '' : 'none'; if (ccToggle) ccToggle.style.display = hasCcBcc ? 'none' : ''; _syncEmailHeaderSummary(); + _stageForwardedSourceAttachments(fields).catch(err => console.error('Forward attachment staging failed:', err)); + } + + function _syncStreamingEmailFields(doc) { + if (!doc) return; + const fields = _parseEmailHeader(doc.content || ''); + const rich = document.getElementById('doc-email-richbody'); + const srcWrap = document.getElementById('doc-editor-wrap'); + const textarea = document.getElementById('doc-editor-textarea'); + if (!rich || rich.style.display === 'none') { + _showEmailFields(doc); + return; + } + + const setValue = (id, value) => { + const el = document.getElementById(id); + const next = value || ''; + if (el && el.value !== next) el.value = next; + }; + setValue('doc-email-to', fields.to); + setValue('doc-email-subject', fields.subject); + setValue('doc-email-in-reply-to', fields.inReplyTo); + setValue('doc-email-references', fields.references); + setValue('doc-email-source-uid', fields.sourceUid || ''); + setValue('doc-email-source-folder', fields.sourceFolder || ''); + setValue('doc-email-cc', fields.cc || ''); + setValue('doc-email-bcc', fields.bcc || ''); + + const unreadBtn = document.getElementById('doc-email-unread-btn'); + if (unreadBtn) unreadBtn.style.display = fields.sourceUid ? '' : 'none'; + const ccRow = document.getElementById('doc-email-cc-row'); + const bccRow = document.getElementById('doc-email-bcc-row'); + const ccToggle = document.getElementById('doc-email-show-cc'); + const hasCcBcc = !!(fields.cc || fields.bcc); + if (ccRow) ccRow.style.display = hasCcBcc ? '' : 'none'; + if (bccRow) bccRow.style.display = hasCcBcc ? '' : 'none'; + if (ccToggle) ccToggle.style.display = hasCcBcc ? 'none' : ''; + + if (srcWrap) srcWrap.style.display = 'none'; + rich.style.display = ''; + _renderStreamingEmailBody(fields.body || ''); + if (doc._originalBody == null) doc._originalBody = fields.body || ''; + _syncEmailHeaderSummary(); } async function _uploadComposeFiles(files) { @@ -2695,6 +2905,47 @@ import * as Modals from './modalManager.js'; _renderComposeAttachments(); } + async function _stageForwardedSourceAttachments(fields) { + const doc = docs.get(activeDocId); + if (!doc || doc.language !== 'email') return; + if (!fields?.forwardAttachments || !fields.sourceUid || !Array.isArray(fields.attachments) || fields.attachments.length === 0) return; + const sourceKey = `${fields.sourceFolder || 'INBOX'}:${fields.sourceUid}:${fields.attachments.map(a => a.index).join(',')}`; + if (doc._forwardedAttachmentSourceKey === sourceKey) return; + doc._forwardedAttachmentSourceKey = sourceKey; + if (!doc._composeAtts) doc._composeAtts = []; + const existingForwarded = new Set(doc._composeAtts.filter(a => a.forwardedSourceKey === sourceKey).map(a => String(a.sourceIndex))); + let added = 0; + for (const att of fields.attachments) { + const sourceIndex = String(att.index); + if (existingForwarded.has(sourceIndex)) continue; + try { + const folderQs = encodeURIComponent(fields.sourceFolder || 'INBOX'); + const res = await fetch(`${API_BASE}/api/email/compose-from-attachment/${encodeURIComponent(fields.sourceUid)}/${encodeURIComponent(att.index)}?folder=${folderQs}`, { + method: 'POST', + credentials: 'same-origin', + }); + const data = await res.json(); + if (!data.success) throw new Error(data.error || 'failed'); + doc._composeAtts.push({ + token: data.token, + filename: data.filename || att.filename, + size: data.size || att.size || 0, + forwardedSourceKey: sourceKey, + sourceIndex, + }); + added += 1; + } catch (err) { + console.error('Failed to stage forwarded attachment:', err); + if (uiModule) uiModule.showError(`Forward attachment failed: ${att.filename || 'attachment'}`); + } + } + if (added) { + _renderComposeAttachments(); + clearTimeout(_autoSaveDebounce); + _autoSaveDebounce = setTimeout(() => { saveDocument({ silent: true }); }, 800); + } + } + async function _handleAttachUpload(e) { const files = e.target.files; e.target.value = ''; // reset for next upload @@ -3117,6 +3368,8 @@ import * as Modals from './modalManager.js'; in_reply_to: inReplyTo || null, references: references || null, attachments: attachments.length > 0 ? attachments : null, account_id: activeAccountId, + source_uid: sourceUid || null, + source_folder: sourceFolder || null, wait_for_delivery: true, }), }); @@ -3166,9 +3419,12 @@ import * as Modals from './modalManager.js'; } // Mark the source email as answered if this was a reply if (sourceUid) { - fetch(`${API_BASE}/api/email/mark-answered/${sourceUid}?folder=${encodeURIComponent(sourceFolder)}`, { method: 'POST' }).catch(() => {}); + _clearEmailLocalDraft(sourceUid, sourceFolder, inReplyTo); + const markParams = new URLSearchParams({ folder: sourceFolder }); + if (data.account_id || activeAccountId) markParams.set('account_id', data.account_id || activeAccountId); + fetch(`${API_BASE}/api/email/mark-answered/${encodeURIComponent(sourceUid)}?${markParams.toString()}`, { method: 'POST' }).catch(() => {}); // Tell the inbox to refresh so the answered state shows - window.dispatchEvent(new CustomEvent('email-answered', { detail: { uid: sourceUid } })); + window.dispatchEvent(new CustomEvent('email-answered', { detail: { uid: sourceUid, folder: sourceFolder, account_id: data.account_id || activeAccountId || null } })); } // Delete the compose document after successful send. It was usually // already detached from the visible tabs so sending can finish in the @@ -3637,6 +3893,7 @@ import * as Modals from './modalManager.js'; const data = await res.json(); if (data.success) { if (uiModule) uiModule.showToast(`Scheduled for ${new Date(localDt).toLocaleString()}`); + _clearCurrentEmailLocalDraft(); cleanup(); // Close the document _closeWithoutDeleting(true); @@ -3800,20 +4057,15 @@ import * as Modals from './modalManager.js'; } - // Detach a doc from its chat session so it stops reappearing in that - // chat: docs with content are unlinked (kept in the library), empty docs - // are deleted. Used by both the tab × and the mobile chip-to-trash close. + // Close a doc tab without breaking its chat association. The chat transcript + // can contain durable document links, so detaching a non-empty doc from the + // session makes it look like the document vanished from that chat. function _detachDocFromSession(docId, { toast = false } = {}) { const doc = docs.get(docId); const hasContent = doc && doc.content && doc.content.trim().length > 0; if (hasContent) { - fetch(`${API_BASE}/api/document/${docId}`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: '' }), - }).then(() => { - if (toast && uiModule) uiModule.showToast('Document unlinked from session'); - }).catch(() => {}); + saveDocument({ silent: true }).catch(() => {}); + if (toast && uiModule) uiModule.showToast('Document closed'); } else { fetch(`${API_BASE}/api/document/${docId}`, { method: 'DELETE' }).catch(() => {}); } @@ -3921,6 +4173,7 @@ import * as Modals from './modalManager.js'; const _rich = document.getElementById('doc-email-richbody'); const _emailBody = (_rich && _rich.style.display !== 'none') ? _rich.innerHTML : textarea.value; doc.content = _buildEmailContent(to, subject, inReplyTo, references, _emailBody, sourceUid, sourceFolder, cc, bcc); + _persistEmailLocalDraftSoon(); } else if (textarea) { // Don't clobber a PDF/form-backed doc's source when the textarea is empty // (it's hidden behind the rendered PDF view, so its value isn't the source @@ -4101,8 +4354,8 @@ import * as Modals from './modalManager.js';