Jake/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-07-14 12:48:03 +00:00
Code Packages Projects Releases Activity

Checkpoint Odysseus local update

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-07-07 00:50:07 +00:00
parent 5f6e6a2c4a
commit 017903de61
66 changed files with 22349 additions and 982 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routes/chat_helpers.py
+16
View File
@@ -14,6 +14,7 @@ from core.database import Session as DBSession, ModelEndpoint
from src.llm_core import normalize_model_id
from src.endpoint_resolver import normalize_base
from src.context_compactor import maybe_compact, trim_for_context
from src.model_context import estimate_tokens
from src.auth_helpers import effective_user
from src.prompt_security import untrusted_context_message
from routes.prefs_routes import _load_for_user as load_prefs_for_user
@@ -99,6 +100,11 @@ class ChatContext:
uprefs: dict
preset: PresetInfo
preprocessed: PreprocessedMessage
context_trimmed: bool = False
context_messages_before_trim: int = 0
context_messages_after_trim: int = 0
context_tokens_before_trim: int = 0
context_tokens_after_trim: int = 0
# Documents auto-created server-side during preprocess (e.g. when an
# attached fillable PDF gets rendered into a markdown editor doc).
# The chat route emits a doc_update SSE event for each before streaming
@@ -777,7 +783,12 @@ async def build_chat_context(
messages, context_length, was_compacted = await maybe_compact(
sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
)
_before_trim_messages = len(messages)
_before_trim_tokens = estimate_tokens(messages)
messages = trim_for_context(messages, context_length)
_after_trim_messages = len(messages)
_after_trim_tokens = estimate_tokens(messages)
_context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
return ChatContext(
preface=preface,
@@ -791,6 +802,11 @@ async def build_chat_context(
uprefs=uprefs,
preset=preset,
preprocessed=preprocessed,
context_trimmed=_context_trimmed,
context_messages_before_trim=_before_trim_messages,
context_messages_after_trim=_after_trim_messages,
context_tokens_before_trim=_before_trim_tokens,
context_tokens_after_trim=_after_trim_tokens,
auto_opened_docs=auto_opened_docs,
uploaded_files=uploaded_files,
)
routes/chat_routes.py
+154 -10
View File
@@ -3,6 +3,7 @@
import asyncio
import json
import os
import re
import time
import logging
from datetime import datetime
@@ -40,7 +41,7 @@ from routes.chat_helpers import (
clean_thinking_for_save,
_enforce_chat_privileges,
)
from src.action_intents import classify_tool_intent as _classify_tool_intent
from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent
from src.tool_policy import build_effective_tool_policy
logger = logging.getLogger(__name__)
@@ -63,6 +64,78 @@ def _stream_set(session_id: str, **fields) -> None:
rec.update(fields)
def _message_plain_text(content: Any) -> str:
if isinstance(content, list):
parts: List[str] = []
for block in content:
if isinstance(block, dict):
text = block.get("text")
if isinstance(text, str):
parts.append(text)
elif isinstance(block, str):
parts.append(block)
return " ".join(parts)
return str(content or "")
def _last_user_plain_text(messages: List[Dict[str, Any]]) -> str:
for msg in reversed(messages or []):
if msg.get("role") == "user":
return _message_plain_text(msg.get("content"))
return ""
def _ensure_current_request_is_latest_user(messages: List[Dict[str, Any]], current_message: str) -> List[Dict[str, Any]]:
"""Defensively keep detached streams grounded on the request that created them."""
current = str(current_message or "").strip()
if not current:
return messages
latest = _last_user_plain_text(messages).strip()
if latest == current or current in latest or latest in current:
return messages
logger.warning(
"[chat_stream] latest user context mismatch; appending current request for model call. latest=%r current=%r",
latest[:120],
current[:120],
)
repaired = list(messages or [])
repaired.append({"role": "user", "content": current})
return repaired
_WEB_FOLLOWUP_RE = re.compile(
r"^\s*(?:(?:can|could|would|will)\s+you\s+)?"
r"(?:check|try\s+again|look(?:\s+now|\s+it\s+up)?|search(?:\s+now|\s+online|\s+it)?|"
r"do\s+it|again)\??\s*$",
re.I,
)
_RECENT_WEB_CONTEXT_RE = re.compile(
r"\b(?:weather|forecast|rain|raining|hourly|news|headlines|rate|exchange|currency|"
r"price|current|latest|search|look\s+up|online)\b",
re.I,
)
def _recent_session_text(sess, limit: int = 8, max_chars: int = 2000) -> str:
history = getattr(sess, "history", None) or getattr(sess, "_history", None) or []
chunks: List[str] = []
for msg in history[-limit:]:
content = getattr(msg, "content", None)
if content is None and isinstance(msg, dict):
content = msg.get("content")
text = _message_plain_text(content).strip()
if text:
chunks.append(text)
return " ".join(chunks)[-max_chars:]
def _is_contextual_web_followup(message: str, sess) -> bool:
"""Treat short retry/check replies as web lookups when recent context was web."""
if not message or not _WEB_FOLLOWUP_RE.search(message):
return False
return bool(_RECENT_WEB_CONTEXT_RE.search(_recent_session_text(sess)))
def _resolve_request_workspace(request, raw_value) -> tuple:
"""Resolve the posted workspace for this request: (workspace, rejected).
@@ -510,6 +583,10 @@ def setup_chat_routes(
# below). Skill extraction should only learn from real agent sessions,
# not chats we quietly promoted for a notes/calendar intent.
user_requested_agent = (chat_mode == "agent")
_search_enabled = (
str(allow_web_search).lower() == "true"
or str(use_web).lower() == "true"
)
# Intent auto-escalation: if the user is clearly asking the assistant
# to create a todo, reminder, or calendar event, promote chat → agent
# for this turn so the LLM has access to manage_notes / manage_calendar.
@@ -527,6 +604,10 @@ def setup_chat_routes(
_tool_intent.category,
_tool_intent.reason,
)
elif chat_mode == "chat" and _search_enabled:
chat_mode = "agent"
auto_escalated = True
logger.info("chat→agent auto-escalation: search enabled")
active_doc_id = form_data.get("active_doc_id", "").strip()
logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}")
@@ -619,6 +700,20 @@ def setup_chat_routes(
400,
"No model selected for this chat. Open the model picker and choose one before sending.",
)
if (
chat_mode == "chat"
and isinstance(message, str)
and (not _tool_intent or not _tool_intent.needs_tools)
and _is_contextual_web_followup(message, sess)
):
_tool_intent = ToolIntent(True, "web", "contextual web lookup follow-up")
chat_mode = "agent"
auto_escalated = True
logger.info(
"chat→agent auto-escalation: category=%s reason=%s",
_tool_intent.category,
_tool_intent.reason,
)
except SessionNotFoundError as e:
raise HTTPException(404, str(e))
except (ValueError, ValidationError):
@@ -789,6 +884,25 @@ def setup_chat_routes(
):
disabled_tools.add("web_search")
disabled_tools.add("web_fetch")
if _explicit_web_intent:
# A direct lookup/search request should not drift into personal
# tools or shell fallbacks. We still keep web_search/web_fetch
# available even when the frontend toggle is stale/falsy because
# the user's words are the stronger signal.
disabled_tools.update({
"bash", "python",
"search_chats", "manage_skills", "manage_memory",
"read_file", "write_file", "edit_file",
"create_document", "edit_document", "update_document",
"send_email", "reply_to_email",
"manage_notes", "manage_calendar", "manage_tasks",
"api_call", "builtin_browser",
})
disabled_tools.discard("web_search")
disabled_tools.discard("web_fetch")
elif _search_enabled:
disabled_tools.discard("web_search")
disabled_tools.discard("web_fetch")
# Nobody/incognito mode: deny tools that would expose the user's
# persistent memory, past chats, or other identity-linked data.
@@ -839,7 +953,10 @@ def setup_chat_routes(
from src.settings import get_setting
_global_disabled = get_setting("disabled_tools", [])
if _global_disabled and isinstance(_global_disabled, list):
explicit_web_allowed = allow_web_search is not None and str(allow_web_search).lower() == "true"
explicit_web_allowed = (
_explicit_web_intent
or (allow_web_search is not None and str(allow_web_search).lower() == "true")
)
if explicit_web_allowed:
disabled_tools.update(t for t in _global_disabled if t not in {"web_search", "web_fetch"})
else:
@@ -1057,13 +1174,16 @@ def setup_chat_routes(
_active_streams.pop(session, None)
return
messages = ctx.messages
messages = _ensure_current_request_is_latest_user(ctx.messages, message)
# Auto-compact notification
if ctx.was_compacted:
yield f"data: {json.dumps({'type': 'compacted', 'context_length': ctx.context_length})}\n\n"
if ctx.context_trimmed and not ctx.was_compacted:
yield f"data: {json.dumps({'type': 'context_trimmed', 'data': {'context_length': ctx.context_length, 'messages_before': ctx.context_messages_before_trim, 'messages_after': ctx.context_messages_after_trim, 'tokens_before': ctx.context_tokens_before_trim, 'tokens_after': ctx.context_tokens_after_trim}})}\n\n"
full_response = ""
thinking_response = ""
last_metrics = None
# Configured fallback chain for the default chat model. Tried in
@@ -1153,7 +1273,9 @@ def setup_chat_routes(
# Forward them so the client can show a thinking
# indicator, but don't fold them into the saved
# reply (mirrors the rewrite path below).
if not data.get("thinking"):
if data.get("thinking"):
thinking_response += data["delta"]
else:
full_response += data["delta"]
_stream_set(session, partial=full_response)
yield chunk
@@ -1173,6 +1295,12 @@ def setup_chat_routes(
_reported_model = last_metrics.get("model")
last_metrics["requested_model"] = _requested_model
last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
if ctx.context_trimmed:
last_metrics["context_trimmed"] = True
last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
if ctx.context_length and last_metrics.get("input_tokens"):
pct = min(round((last_metrics["input_tokens"] / ctx.context_length) * 100, 1), 100.0)
last_metrics["context_percent"] = pct
@@ -1215,8 +1343,11 @@ def setup_chat_routes(
}
yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
if full_response:
_metrics_to_save = dict(last_metrics or {})
if thinking_response.strip() and not _metrics_to_save.get("thinking"):
_metrics_to_save["thinking"] = thinking_response.strip()
_saved_id = save_assistant_response(
sess, session_manager, session, full_response, last_metrics,
sess, session_manager, session, full_response, _metrics_to_save,
character_name=ctx.preset.character_name,
web_sources=web_sources,
rag_sources=ctx.rag_sources,
@@ -1229,7 +1360,7 @@ def setup_chat_routes(
yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
run_post_response_tasks(
sess, session_manager, session, message, full_response,
last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
_metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
incognito=incognito, compare_mode=compare_mode,
character_name=ctx.preset.character_name,
owner=_user,
@@ -1281,7 +1412,9 @@ def setup_chat_routes(
_max_rounds = max(1, min(_max_rounds, 200))
_forced_tools = None
if allow_web_search is not None and str(allow_web_search).lower() == "true":
if _explicit_web_intent:
_forced_tools = {"web_search", "web_fetch"}
elif _search_enabled:
_forced_tools = {"web_search", "web_fetch"}
async for chunk in stream_agent_loop(
@@ -1315,7 +1448,9 @@ def setup_chat_routes(
# Reasoning tokens arrive flagged thinking:true.
# Forward them for the live indicator, but keep
# them out of the saved reply (same as chat mode).
if not data.get("thinking"):
if data.get("thinking"):
thinking_response += data["delta"]
else:
full_response += data["delta"]
_stream_set(session, partial=full_response)
yield chunk
@@ -1353,6 +1488,12 @@ def setup_chat_routes(
_reported_model = last_metrics.get("model")
last_metrics["requested_model"] = last_metrics.get("requested_model") or _requested_model
last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model
if ctx.context_trimmed:
last_metrics["context_trimmed"] = True
last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim
last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim
last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim
last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim
yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n'
except json.JSONDecodeError:
yield chunk
@@ -1362,8 +1503,11 @@ def setup_chat_routes(
_has_tool_events = bool((last_metrics or {}).get("tool_events"))
if full_response or _has_tool_events:
_response_to_save = full_response or "Done."
_metrics_to_save = dict(last_metrics or {})
if thinking_response.strip() and not _metrics_to_save.get("thinking"):
_metrics_to_save["thinking"] = thinking_response.strip()
_saved_id = save_assistant_response(
sess, session_manager, session, _response_to_save, last_metrics,
sess, session_manager, session, _response_to_save, _metrics_to_save,
character_name=ctx.preset.character_name,
web_sources=web_sources,
rag_sources=ctx.rag_sources,
@@ -1374,7 +1518,7 @@ def setup_chat_routes(
yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n'
run_post_response_tasks(
sess, session_manager, session, message, _response_to_save,
last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
_metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager,
incognito=incognito, compare_mode=compare_mode,
character_name=ctx.preset.character_name,
agent_rounds=_agent_rounds,
routes/contacts_routes.py
+28 -16
View File
@@ -394,21 +394,25 @@ def _resolve_resource_url(uid: str) -> str:
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str, address: str = "") -> bool:
def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
"""Add a new contact via CardDAV or local contacts."""
email = (email or "").strip()
phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = (email or "").strip().lower()
email_l = email.lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
contacts.append(_normalize_contact({"name": name, "emails": [email], "address": address}))
if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
return True
contacts.append(_normalize_contact({"name": name, "emails": [email] if email else [], "phones": phone_list, "address": address}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid, address=address)
vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
try:
url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
auth = None
@@ -759,26 +763,34 @@ def setup_contacts_routes():
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
phone = (data.get("phone") or "").strip()
phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
if phone and phone not in phones:
phones.insert(0, phone)
address = (data.get("address") or "").strip()
if not email:
return {"success": False, "error": "Email required"}
# Check if already exists by email
if email:
contacts = _fetch_contacts()
for c in contacts:
if email.lower() in [e.lower() for e in c["emails"]]:
return {"success": True, "message": "Already exists", "contact": c}
if not name:
if not name and email:
name = email.split("@")[0]
if not name and not email and not phones and not address:
return {"success": False, "error": "Name, email, phone, or address required"}
if not name:
name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
# Check if already exists by email or phone.
contacts = _fetch_contacts()
for c in contacts:
if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
return {"success": True, "message": "Already exists", "contact": c}
if phones and any(p in (c.get("phones") or []) for p in phones):
return {"success": True, "message": "Already exists", "contact": c}
create_params = inspect.signature(_create_contact).parameters
if len(create_params) >= 3:
if "phones" in create_params:
ok = _create_contact(name, email, address, phones=phones)
elif len(create_params) >= 3:
ok = _create_contact(name, email, address)
else:
ok = _create_contact(name, email)
# If a phone was provided, do an immediate update to thread it
# through (the simple _create_contact signature only takes name +
# email + address; phones happen via update).
if ok and phone:
if ok and phones and "phones" not in create_params:
try:
fresh = _fetch_contacts(force=True)
created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
@@ -786,7 +798,7 @@ def setup_contacts_routes():
_update_contact(
created["uid"], name,
created.get("emails", []),
[phone],
phones,
address,
)
except Exception:
routes/cookbook_helpers.py
+52 -12
View File
@@ -439,15 +439,30 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
" if f.is_file(): nf += 1; sz += f.stat().st_size",
" if f.name.endswith('.incomplete'): ic = True",
" snap = os.path.join(cache, d, 'snapshots')",
" # Windows HF cache stores files directly in snapshots/; blobs/ may be empty.",
" # Fallback: scan snapshots for real files when blobs yielded nothing.",
" if sz == 0 and os.path.isdir(snap):",
" def snapshot_size():",
" total, count, incomplete = 0, 0, False",
" seen_real = set()",
" for sd in os.listdir(snap):",
" sf = os.path.join(snap, sd)",
" if not os.path.isdir(sf): continue",
" for f in os.scandir(sf):",
" if f.is_file(): nf += 1; sz += f.stat().st_size",
" if f.name.endswith('.incomplete'): ic = True",
" for root, dirs, fns in safe_walk(sf):",
" for fn in fns:",
" fp = os.path.join(root, fn)",
" if fn.endswith('.incomplete'): incomplete = True",
" try:",
" real = os.path.realpath(fp)",
" if real in seen_real: continue",
" seen_real.add(real)",
" total += os.path.getsize(real)",
" count += 1",
" except Exception:",
" pass",
" return total, count, incomplete",
" # Some HF caches (macOS/MLX/Xet-style) keep blobs elsewhere or expose",
" # snapshot symlinks only. Size snapshots too when blob accounting is empty.",
" if sz == 0 and os.path.isdir(snap):",
" sz2, nf2, ic2 = snapshot_size()",
" sz, nf, ic = sz2, nf2, ic or ic2",
" is_diffusion = False; gguf_files = []",
" if os.path.isdir(snap):",
" for sd in os.listdir(snap):",
@@ -471,7 +486,18 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
" add('/app/.cache/huggingface/hub')",
f" add({add_hf_cache!r})" if add_hf_cache else "",
" return candidates",
"def normalize_model_dir(p):",
" p = os.path.expanduser((p or '').strip())",
" if not p: return p",
" if os.path.isdir(p) or os.path.isabs(p): return p",
" # Users often paste Linux absolute paths without the leading slash.",
" # Treat home/<user>/... as /home/<user>/... so remote scans work.",
" if p.startswith(('home/', 'mnt/', 'media/', 'data/', 'opt/', 'srv/', 'var/')):",
" prefixed = '/' + p",
" if os.path.isdir(prefixed): return prefixed",
" return p",
"def scan_dir(p):",
" p = normalize_model_dir(p)",
" if not os.path.isdir(p) or not safe_path(p): return",
" for d in sorted(os.listdir(p)):",
" if d.startswith('.'): continue",
@@ -541,7 +567,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
"scan_ollama()",
]
for model_dir in model_dirs or []:
lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))")
lines.append(f"scan_dir({model_dir!r})")
lines.append("print(json.dumps(models))")
return "\n".join(lines) + "\n"
@@ -1261,13 +1287,14 @@ def _diagnose_serve_output(text: str) -> dict | None:
[{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
),
(
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
r"Could not load any common_ops library|"
r"Please ensure sgl_kernel is properly installed",
"SGLang native dependencies are missing on this server.",
"SGLang native kernel/runtime is missing or mismatched on this server.",
[
{"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
{"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
{"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
{"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
],
),
(
@@ -1275,6 +1302,19 @@ def _diagnose_serve_output(text: str) -> dict | None:
"SGLang is not installed or not in PATH on this server.",
[{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
),
(
r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
"MLX LM is not installed on this server.",
[{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
),
(
r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
"MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
[
{"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
{"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
],
),
# System build deps come BEFORE the generic llama.cpp catch-all so
# cmake / build-essential / git missing → a specific OS-package
# remediation instead of "install llama-cpp-python[server]" (which
routes/cookbook_routes.py
+639 -87
View File
@@ -58,6 +58,7 @@ from routes.cookbook_helpers import (
_diagnose_serve_output, run_ssh_command_async,
_ollama_bind_from_cmd, _pip_install_fallback_chain, _pip_install_no_cache,
_user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd,
_append_pip_install_runner_lines, _pip_install_command_without_break_system_packages,
_normalize_llama_cpp_python_cache_types,
ModelDownloadRequest, ServeRequest,
)
@@ -71,6 +72,72 @@ _HF_TOKEN_STATUS_SNIPPET = (
'fi'
)
def _venv_root_from_serve_cmd(cmd: str) -> str:
"""Best-effort venv root from an absolute venv python in a serve command."""
try:
parts = shlex.split(cmd or "")
except Exception:
parts = (cmd or "").split()
for part in parts:
if re.search(r"/bin/python(?:3(?:\.\d+)?)?$", part or ""):
return re.sub(r"/bin/python(?:3(?:\.\d+)?)?$", "", part)
return ""
def _append_venv_nvidia_library_path_lines(lines: list[str], *, cmd: str = "") -> None:
"""Expose NVIDIA CUDA runtime wheels bundled inside the active venv.
SGLang/vLLM wheels can depend on CUDA libraries shipped as Python packages
under site-packages/nvidia. Activating the venv puts Python packages on
sys.path, but the dynamic loader still cannot find libraries such as
libnvrtc.so.13 unless those package lib dirs are on LD_LIBRARY_PATH.
"""
venv_root = _venv_root_from_serve_cmd(cmd)
lines.append(f'_ODY_VENV_FOR_LIBS="${{VIRTUAL_ENV:-{_bash_squote(venv_root)}}}"')
lines.append('if [ -n "$_ODY_VENV_FOR_LIBS" ] && [ -d "$_ODY_VENV_FOR_LIBS" ]; then')
lines.append(' for _ody_nvlib in "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cu13/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cu12/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cuda_nvrtc/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cuda_runtime/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cublas/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cudnn/lib; do')
lines.append(' [ -d "$_ody_nvlib" ] && export LD_LIBRARY_PATH="$_ody_nvlib:${LD_LIBRARY_PATH:-}"')
lines.append(' done')
lines.append('fi')
def _serve_port_from_cmd(cmd: str) -> str:
m = re.search(r"--port(?:=|\s+)(\d+)", cmd or "")
return m.group(1) if m else ""
def _append_openai_port_preflight_lines(lines: list[str], *, cmd: str, expected_model: str) -> None:
port = _serve_port_from_cmd(cmd)
if not port:
return
lines.append(f"ODYSSEUS_SERVE_PORT='{_bash_squote(port)}'")
lines.append(f"ODYSSEUS_EXPECTED_MODEL='{_bash_squote(expected_model)}'")
lines.append("if [ -n \"$ODYSSEUS_SERVE_PORT\" ]; then")
lines.append(" python3 - \"$ODYSSEUS_SERVE_PORT\" \"$ODYSSEUS_EXPECTED_MODEL\" <<'PY'")
lines.append("import json, sys, urllib.request")
lines.append("port = sys.argv[1]")
lines.append("expected = (sys.argv[2] or '').strip()")
lines.append("url = f'http://127.0.0.1:{port}/v1/models'")
lines.append("try:")
lines.append(" with urllib.request.urlopen(url, timeout=1.5) as r:")
lines.append(" data = json.loads(r.read().decode('utf-8', 'replace') or '{}')")
lines.append("except Exception:")
lines.append(" raise SystemExit(0)")
lines.append("models = [str(x.get('id') or '') for x in data.get('data', []) if isinstance(x, dict)]")
lines.append("def base(s): return s.lower().split('/')[-1]")
lines.append("match = bool(expected) and any((m.lower() == expected.lower() or base(m) == base(expected) or base(expected) in m.lower() or base(m) in expected.lower()) for m in models)")
lines.append("print(f'ERROR: Port {port} is already serving {models or [\"unknown\"]}.')")
lines.append("if expected and not match:")
lines.append(" print(f'ERROR: Cookbook was about to launch {expected}, but this port is occupied by a different model. Stop the old server or choose another port.')")
lines.append("else:")
lines.append(" print('ERROR: Stop the existing server or choose another port before launching a duplicate serve.')")
lines.append("raise SystemExit(98)")
lines.append("PY")
lines.append(" _ody_port_ec=$?")
lines.append(" if [ \"$_ody_port_ec\" -ne 0 ]; then ODYSSEUS_PREFLIGHT_EXIT=\"$_ody_port_ec\"; fi")
lines.append("fi")
_OLLAMA_SIDECAR_CONTAINERS = {"ollama-test", "ollama-rocm"}
_UNSAFE_DOCKER_EXEC_CHARS = frozenset(";&|<>$`\r\n")
_SAFE_OLLAMA_MODEL_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$")
@@ -145,7 +212,7 @@ async def _remote_binary_available(
if windows:
check = f'powershell -NoProfile -Command "if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}"'
else:
check = f"command -v {shlex.quote(binary)} >/dev/null 2>&1"
check = f'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; command -v {shlex.quote(binary)} >/dev/null 2>&1'
try:
proc = await asyncio.create_subprocess_exec(
"ssh",
@@ -165,6 +232,45 @@ async def _remote_binary_available(
return False
def _remote_posix_path_prefix() -> str:
return 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; '
def _remote_tmux_command(*args: str) -> str:
"""Shell command for remote tmux when non-login SSH has a thin PATH."""
tmux = (
'ODYSSEUS_TMUX="$(command -v tmux '
'|| command -v /opt/homebrew/bin/tmux '
'|| command -v /usr/local/bin/tmux '
'|| command -v /usr/bin/tmux '
'|| true)"; '
'if [ -z "$ODYSSEUS_TMUX" ]; then echo "tmux not found" >&2; exit 127; fi; '
)
quoted = " ".join(shlex.quote(str(arg)) for arg in args)
return f'{_remote_posix_path_prefix()}{tmux}"$ODYSSEUS_TMUX" {quoted}'
def _remote_tmux_launch_command(session_id: str, runner: str) -> str:
"""Shell command that chmods a runner and starts it in remote tmux."""
tmux = (
'ODYSSEUS_TMUX="$(command -v tmux '
'|| command -v /opt/homebrew/bin/tmux '
'|| command -v /usr/local/bin/tmux '
'|| command -v /usr/bin/tmux '
'|| true)"; '
'if [ -z "$ODYSSEUS_TMUX" ]; then echo "tmux not found" >&2; exit 127; fi; '
)
sid = shlex.quote(str(session_id))
runner_q = shlex.quote(str(runner))
runner_exec = shlex.quote(f"./{runner}")
return (
f'{_remote_posix_path_prefix()}{tmux}'
f'chmod +x {runner_q} && '
f'"$ODYSSEUS_TMUX" set-option -g history-limit 100000 2>/dev/null; '
f'"$ODYSSEUS_TMUX" new-session -d -s {sid} {runner_exec}'
)
async def _binary_available(
binary: str,
remote: str | None,
@@ -366,13 +472,14 @@ def setup_cookbook_routes() -> APIRouter:
[{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
),
(
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops)|"
r"(Python\.h|libnuma\.so\.1|common_ops)[\s\S]*sgl_kernel|"
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
r"Could not load any common_ops library|"
r"Please ensure sgl_kernel is properly installed",
"SGLang native dependencies are missing on this server.",
"SGLang native kernel/runtime is missing or mismatched on this server.",
[
{"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
{"label": "upgrade sglang-kernel after OS packages are installed", "op": "manual"},
{"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
{"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
],
),
(
@@ -380,6 +487,19 @@ def setup_cookbook_routes() -> APIRouter:
"SGLang is not installed or not in PATH on this server.",
[{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
),
(
r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
"MLX LM is not installed on this server.",
[{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
),
(
r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
"MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
[
{"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
{"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
],
),
# System build deps come BEFORE the generic llama.cpp catch-all
# so cmake / build-essential / git missing → a specific OS-package
# remediation instead of "install llama-cpp-python[server]" (which
@@ -542,6 +662,89 @@ def setup_cookbook_routes() -> APIRouter:
add_bool("--enable-expert-parallel")
return shlex.join(env_parts + body)
def _normalize_deepseek_v4_sglang_cmd(cmd: str) -> str:
"""Patch stale DeepSeek-V4 SGLang commands into the safer local form.
The browser command builder already emits these flags, but saved presets,
running-row retries, and old tabs can still submit a pre-fix command to
/api/model/serve. Normalize server-side so the tmux runner does not keep
relaunching DeepSeek-V4 with the known CUDA-graph crash shape.
"""
cmd_lower = (cmd or "").lower()
if (
not cmd
or "sglang.launch_server" not in cmd_lower
or "deepseek-v4" not in cmd_lower
):
return cmd
try:
parts = shlex.split(cmd)
except ValueError:
return cmd
env_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
env_parts = [p for p in parts if env_re.match(p)]
body = [p for p in parts if not env_re.match(p)]
def add_env(key: str, value: str) -> None:
if not any(p.startswith(f"{key}=") for p in env_parts):
env_parts.append(f"{key}={value}")
def has_flag(flag: str) -> bool:
return any(p == flag or p.startswith(flag + "=") for p in body)
def flag_value(flag: str) -> str | None:
for i, part in enumerate(body):
if part == flag:
return body[i + 1] if i + 1 < len(body) else ""
if part.startswith(flag + "="):
return part.split("=", 1)[1]
return None
def set_flag(flag: str, value: str) -> None:
for i, part in enumerate(body):
if part == flag:
if i + 1 < len(body):
body[i + 1] = value
else:
body.append(value)
return
if part.startswith(flag + "="):
body[i] = f"{flag}={value}"
return
body.extend([flag, value])
def remove_flag(flag: str) -> None:
i = 0
while i < len(body):
part = body[i]
if part == flag:
del body[i:i + 2]
continue
if part.startswith(flag + "="):
del body[i]
continue
i += 1
add_env("SGLANG_DSV4_COMPRESS_STATE_DTYPE", "bf16")
mem_fraction = flag_value("--mem-fraction-static")
try:
mem_fraction_num = float(mem_fraction) if mem_fraction not in (None, "") else None
except (TypeError, ValueError):
mem_fraction_num = None
if mem_fraction in (None, "", "0.90", "0.9") or (
mem_fraction_num is not None and mem_fraction_num < 0.76
):
set_flag("--mem-fraction-static", "0.80")
if not has_flag("--reasoning-parser"):
set_flag("--reasoning-parser", "deepseek-v4")
if not has_flag("--tool-call-parser"):
set_flag("--tool-call-parser", "deepseekv4")
if not has_flag("--cuda-graph-backend-decode"):
remove_flag("--cuda-graph-max-bs-decode")
set_flag("--cuda-graph-backend-decode", "disabled")
return shlex.join(env_parts + body)
def _cookbook_ssh_dir() -> Path:
# The Docker image keeps cookbook keys under /app/.ssh; that path only
# exists inside the container. On Windows (and any non-container host)
@@ -555,12 +758,124 @@ def setup_cookbook_routes() -> APIRouter:
def _cookbook_ssh_key_path() -> Path:
return _cookbook_ssh_dir() / "id_ed25519"
def _ssh_known_host_name(host: str) -> str:
"""Return the host part OpenSSH stores in known_hosts.
Cookbook accepts `user@host` for convenience, but known_hosts entries
are keyed by host, not username.
"""
return (host or "").rsplit("@", 1)[-1]
def _known_hosts_targets(host: str, ssh_port: str | None = None) -> list[str]:
name = _ssh_known_host_name(host)
targets = [name]
if ssh_port and ssh_port != "22":
targets.insert(0, f"[{name}]:{ssh_port}")
return [t for t in targets if t]
def _ssh_host_key_changed(stderr_txt: str) -> bool:
text = stderr_txt or ""
return (
"REMOTE HOST IDENTIFICATION HAS CHANGED" in text
or "Host key verification failed" in text and "Offending" in text
)
async def _repair_cookbook_known_host(host: str, ssh_port: str | None = None) -> tuple[bool, str]:
"""Refresh Odysseus' own known_hosts entry for a validated Cookbook host.
This is intentionally scoped to Cookbook SSH targets and only called
after OpenSSH reports a changed host key. It fixes container-local
known_hosts drift without asking the user to run ssh-keygen manually.
"""
known_hosts = _cookbook_ssh_dir() / "known_hosts"
known_hosts.parent.mkdir(parents=True, exist_ok=True)
known_hosts.touch(mode=0o600, exist_ok=True)
safe_chmod(known_hosts, 0o600)
ssh_keygen = which_tool("ssh-keygen") or "ssh-keygen"
ssh_keyscan = which_tool("ssh-keyscan") or "ssh-keyscan"
removed_chunks: list[str] = []
for target in _known_hosts_targets(host, ssh_port):
proc = await asyncio.create_subprocess_exec(
ssh_keygen,
"-f",
str(known_hosts),
"-R",
target,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
removed_chunks.append((stdout or stderr).decode("utf-8", errors="replace").strip())
scan_args = [ssh_keyscan, "-H", "-t", "ed25519,ecdsa,rsa"]
if ssh_port and ssh_port != "22":
scan_args.extend(["-p", ssh_port])
scan_args.append(_ssh_known_host_name(host))
proc = await asyncio.create_subprocess_exec(
*scan_args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=8)
except asyncio.TimeoutError:
proc.kill()
await proc.communicate()
return False, "ssh-keyscan timed out while refreshing known_hosts"
if proc.returncode != 0 or not stdout.strip():
detail = (stderr or stdout).decode("utf-8", errors="replace").strip()
return False, detail or "ssh-keyscan returned no host keys"
with known_hosts.open("ab") as f:
if known_hosts.stat().st_size > 0:
f.write(b"\n")
f.write(stdout.strip() + b"\n")
safe_chmod(known_hosts, 0o600)
return True, "\n".join(chunk for chunk in removed_chunks if chunk) or "known_hosts refreshed"
def _read_cookbook_public_key() -> str:
pub = _cookbook_ssh_key_path().with_suffix(".pub")
if not pub.exists():
return ""
return pub.read_text(encoding="utf-8", errors="replace").strip()
def _server_env_prefix_for_download(remote_host: str | None) -> str | None:
"""Recover a server venv/conda activation for stale download clients.
Older browser bundles could submit /api/model/download without
env_prefix even when the selected server profile had an envPath. The
remote runner would then use system python and exit before hf download.
Resolve the server profile by host here so downloads remain correct even
if the user has a cached JS bundle.
"""
if not remote_host or not _cookbook_state_path.exists():
return None
try:
state = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
except Exception:
return None
env_state = state.get("env") if isinstance(state, dict) else {}
servers = env_state.get("servers") if isinstance(env_state, dict) else []
if not isinstance(servers, list):
return None
selected = None
for server in servers:
if isinstance(server, dict) and (server.get("host") or "").strip() == remote_host:
selected = server
break
if not selected:
return None
env = (selected.get("env") or "none").strip().lower()
env_path = (selected.get("envPath") or "").strip()
if not env_path:
return None
if env == "venv" or (env in {"", "none"} and re.search(r"(?:^|/)(?:\.?venv|env)(?:/|$)|/bin/activate$", env_path, re.I)):
activate = env_path if env_path.endswith("/bin/activate") else env_path.rstrip("/") + "/bin/activate"
return "source " + shlex.quote(activate)
if env == "conda":
return 'eval "$(conda shell.bash hook)" && conda activate ' + shlex.quote(env_path)
return None
@router.get("/api/cookbook/ssh-key")
async def get_cookbook_ssh_key(request: Request):
require_admin(request)
@@ -708,6 +1023,8 @@ def setup_cookbook_routes() -> APIRouter:
req.local_dir = _validate_local_dir(req.local_dir)
req.hf_token = "" if is_ollama_download else (req.hf_token or _load_stored_hf_token())
_validate_token(req.hf_token)
if req.remote_host and not req.env_prefix:
req.env_prefix = _server_env_prefix_for_download(req.remote_host)
TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
session_id = f"cookbook-{uuid.uuid4().hex[:8]}"
wrapper_script = TMUX_LOG_DIR / f"{session_id}.sh"
@@ -725,9 +1042,10 @@ def setup_cookbook_routes() -> APIRouter:
# Build the hf download command. Redirection to suppress the interactive
# "update available? [Y/n]" prompt is added per-platform further down
# (< /dev/null on bash, $null | on PowerShell).
hf_cmd = f"hf download {req.repo_id}"
hf_download_args = f"download {shlex.quote(req.repo_id)}"
if req.include:
hf_cmd += f" --include '{req.include}'"
hf_download_args += f" --include {shlex.quote(req.include)}"
hf_cmd = f"hf {hf_download_args}"
ollama_cmd = f"ollama pull {shlex.quote(req.repo_id)}"
# Build the shell wrapper — runs hf download directly in tmux (which is a TTY)
@@ -879,6 +1197,8 @@ def setup_cookbook_routes() -> APIRouter:
)
# Ensure pip-user scripts (e.g. hf CLI installed via --user) are on PATH
runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
runner_lines.append('ODYSSEUS_PY="$(command -v python3 || command -v python || true)"')
runner_lines.append('if [ -z "$ODYSSEUS_PY" ]; then echo "ERROR: python3/python not found on this server."; exit 127; fi')
# Install hf CLI + optional hf_transfer best-effort. Retries disable
# hf_transfer because the Rust parallel path is fast but has been
# flaky near the end of very large multi-file downloads.
@@ -894,13 +1214,16 @@ def setup_cookbook_routes() -> APIRouter:
runner_lines.append('fi')
runner_lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')
else:
runner_lines.append(f"command -v hf >/dev/null 2>&1 || {_pip_install_fallback_chain('huggingface_hub', python_cmd='pip', upgrade=True)}")
runner_lines.append(f"command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1 || {_pip_install_fallback_chain('huggingface_hub', python_cmd='\"$ODYSSEUS_PY\" -m pip', upgrade=True)}")
runner_lines.append('hash -r 2>/dev/null || true')
runner_lines.append('ODYSSEUS_HF_CLI="$(command -v hf || command -v huggingface-cli || true)"')
runner_lines.append('if [ -z "$ODYSSEUS_HF_CLI" ]; then echo "ERROR: HF CLI not found after installing huggingface_hub."; exit 127; fi')
if req.disable_hf_transfer:
runner_lines.append("export HF_HUB_ENABLE_HF_TRANSFER=0")
runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=4")
else:
runner_lines.append(f"python3 -c 'import hf_transfer' 2>/dev/null || {_pip_install_fallback_chain('hf_transfer', python_cmd='pip')}")
runner_lines.append("python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1")
runner_lines.append(f"\"$ODYSSEUS_PY\" -c 'import hf_transfer' 2>/dev/null || {_pip_install_fallback_chain('hf_transfer', python_cmd='\"$ODYSSEUS_PY\" -m pip')}")
runner_lines.append("\"$ODYSSEUS_PY\" -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1")
runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8")
# Surface whether the HF token actually reached THIS server, so a gated
# download's "not authorized" failure can be told apart from a missing
@@ -915,22 +1238,7 @@ def setup_cookbook_routes() -> APIRouter:
if is_ollama_download:
runner_lines.append(' eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null')
else:
runner_lines.append(' if command -v hf &>/dev/null; then')
runner_lines.append(f' {hf_cmd} < /dev/null')
runner_lines.append(' elif python3 -c "import huggingface_hub" 2>/dev/null; then')
runner_lines.append(' [ $_attempt -eq 1 ] && echo "hf CLI not found, using Python huggingface_hub..."')
runner_lines.append(f' python3 -c "import os; from huggingface_hub import snapshot_download; snapshot_download(\'{req.repo_id}\'{_dl_pyarg}, max_workers={mw})"')
runner_lines.append(' else')
runner_lines.append(' echo "Installing huggingface-hub and dependencies..."')
runner_lines.append(' pip install --no-deps -q huggingface-hub 2>/dev/null')
if req.disable_hf_transfer:
runner_lines.append(' pip install -q filelock fsspec packaging pyyaml tqdm typer httpx requests 2>/dev/null')
runner_lines.append(' export HF_HUB_ENABLE_HF_TRANSFER=0')
else:
runner_lines.append(' pip install -q filelock fsspec packaging pyyaml tqdm typer httpx requests hf_transfer 2>/dev/null')
runner_lines.append(" python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1")
runner_lines.append(f' python3 -c "import os; from huggingface_hub import snapshot_download; snapshot_download(\'{req.repo_id}\'{_dl_pyarg}, max_workers={mw})"')
runner_lines.append(' fi')
runner_lines.append(f' "$ODYSSEUS_HF_CLI" {hf_download_args} < /dev/null')
runner_lines.append(' _ec=$?')
runner_lines.append(' if [ $_ec -eq 0 ]; then break; fi')
runner_lines.append(' if [ $_attempt -lt $_max_retries ]; then')
@@ -953,7 +1261,7 @@ def setup_cookbook_routes() -> APIRouter:
_spf = f"-p {_port} " if _port and _port != "22" else ""
setup_cmd = (
f"scp -O {_pf}-q '{runner_path}' {remote}:{remote_runner} && "
f"ssh {_spf}{remote} 'chmod +x {remote_runner} && tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} \"./{remote_runner}\"'"
f"ssh {_spf}{remote} {shlex.quote(_remote_tmux_launch_command(session_id, remote_runner))}"
)
else:
# Local: run hf download in the background (tmux on POSIX, a detached
@@ -1041,49 +1349,64 @@ def setup_cookbook_routes() -> APIRouter:
for d in model_dir.split(','):
d = d.strip()
if d:
if d.startswith(("home/", "mnt/", "media/", "data/", "opt/", "srv/", "var/")):
d = "/" + d
model_dirs.append(d)
paths_code = _cached_model_scan_script(model_dirs)
scan_py = TMUX_LOG_DIR / "scan_cache.py"
scan_py.write_text(paths_code, encoding="utf-8")
if host:
_pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
if platform == "windows":
# Windows: use 'python' and pipe via stdin with double-quote wrapping
cmd = f'ssh {_pf}{host} "python -" < \'{scan_py}\''
async def _run_cached_scan_once():
if host:
_ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 "
_pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
if platform == "windows":
# Windows: use 'python' and pipe via stdin with double-quote wrapping
cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\''
else:
cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'"
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=str(Path.home()),
)
else:
cmd = f"ssh {_pf}{host} 'python3 -' < '{scan_py}'"
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=str(Path.home()),
)
else:
# LOCAL scan: use sys.executable (the venv Python Odysseus is already
# running under) — it's guaranteed real Python on all platforms.
# Falling back to which_tool on Windows risks hitting the Microsoft
# Store stub alias for "python3"/"python", which prints
# "Python was not found; run without arguments to install from the
# Microsoft Store" and exits 9009, producing empty stdout and a
# JSON parse error. sys.executable bypasses PATH entirely.
local_py = sys.executable or (
which_tool("python3") or which_tool("python")
or which_tool("py") or "python"
)
proc = await asyncio.create_subprocess_exec(
local_py, str(scan_py),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=str(Path.home()),
)
stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=60)
# LOCAL scan: use sys.executable (the venv Python Odysseus is already
# running under) — it's guaranteed real Python on all platforms.
# Falling back to which_tool on Windows risks hitting the Microsoft
# Store stub alias for "python3"/"python", which prints
# "Python was not found; run without arguments to install from the
# Microsoft Store" and exits 9009, producing empty stdout and a
# JSON parse error. sys.executable bypasses PATH entirely.
local_py = sys.executable or (
which_tool("python3") or which_tool("python")
or which_tool("py") or "python"
)
proc = await asyncio.create_subprocess_exec(
local_py, str(scan_py),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=str(Path.home()),
)
return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode
(stdout_b, stderr_b), returncode = await _run_cached_scan_once()
stderr_txt = stderr_b.decode(errors="replace").strip()
stdout_txt = stdout_b.decode(errors="replace").strip()
if proc.returncode != 0:
msg = stderr_txt or f"Cached model scan failed with exit code {proc.returncode}"
logger.warning(f"Cached model scan failed host={host or 'local'} rc={proc.returncode}: {msg[:500]}")
if host and returncode != 0 and _ssh_host_key_changed(stderr_txt):
ok, detail = await _repair_cookbook_known_host(host, ssh_port)
if ok:
logger.info("Repaired Cookbook known_hosts for %s after host-key-change scan failure", host)
(stdout_b, stderr_b), returncode = await _run_cached_scan_once()
stderr_txt = stderr_b.decode(errors="replace").strip()
stdout_txt = stdout_b.decode(errors="replace").strip()
else:
logger.warning("Failed to repair Cookbook known_hosts for %s: %s", host, detail[:300])
if returncode != 0:
msg = stderr_txt or f"Cached model scan failed with exit code {returncode}"
logger.warning(f"Cached model scan failed host={host or 'local'} rc={returncode}: {msg[:500]}")
return {"models": [], "host": host or "local", "error": msg}
models = []
@@ -1270,7 +1593,7 @@ def setup_cookbook_routes() -> APIRouter:
ssh_args = ["ssh"]
if ssh_port and ssh_port != "22":
ssh_args.extend(["-p", str(ssh_port)])
capture_cmd = ssh_args + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"]
capture_cmd = ssh_args + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-2000")]
else:
capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"]
@@ -1400,12 +1723,27 @@ def setup_cookbook_routes() -> APIRouter:
short_name = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id
display_name = short_name or "Local model"
is_mlx_deepseek_v4 = (
"mlx_lm.server" in (req.cmd or "")
and "deepseek-v4" in ((req.repo_id or "") + " " + (req.cmd or "")).lower()
)
mlx_shim_model_id = ""
if is_mlx_deepseek_v4 and short_name:
home_match = re.search(r"((?:/Users|/home)/[^/\s'\"]+)", req.cmd or "")
remote_home = home_match.group(1) if home_match else ""
if remote_home:
mlx_shim_model_id = f"{remote_home}/.cache/odysseus/mlx-shims/{short_name}"
# If the serve command opts models into OpenAI tool-calling, record it so
# agent_loop trusts emitted tool_calls instead of the name heuristic.
is_ollama_endpoint = "ollama" in (req.cmd or "").lower()
supports_tools = True if "--enable-auto-tool-choice" in req.cmd else None
pinned_models = [req.repo_id] if is_ollama_endpoint and req.repo_id else []
# Pin the model the user launched for every Cookbook-created LLM
# endpoint, not just Ollama. Some OpenAI-compatible servers report a
# deployment alias from /v1/models, and a stale server can answer on the
# same port while the new launch failed. Keeping the requested model id
# pinned makes the picker reflect the actual launch intent.
pinned_models = [mlx_shim_model_id] if mlx_shim_model_id else ([req.repo_id] if req.repo_id else [])
db = SessionLocal()
try:
@@ -1417,11 +1755,20 @@ def setup_cookbook_routes() -> APIRouter:
existing.name = display_name
existing.endpoint_kind = "local"
existing.model_refresh_mode = "auto"
if pinned_models:
try:
existing_pinned = json.loads(existing.pinned_models or "[]")
except Exception:
existing_pinned = []
merged_pinned = []
for mid in [*existing_pinned, *pinned_models]:
if mid and mid not in merged_pinned:
merged_pinned.append(mid)
existing.pinned_models = json.dumps(merged_pinned) if merged_pinned else None
if is_ollama_endpoint:
existing.endpoint_kind = "ollama"
if pinned_models:
existing.cached_models = json.dumps(pinned_models)
existing.pinned_models = json.dumps(pinned_models)
if supports_tools is not None:
existing.supports_tools = supports_tools
db.commit()
@@ -1430,12 +1777,17 @@ def setup_cookbook_routes() -> APIRouter:
# serves right now (the URL may have stayed the same but the
# model behind it changed across launches).
try:
from routes.model_routes import _probe_endpoint
import json as _json2
probed = _probe_endpoint(base_url, existing.api_key, timeout=5)
if probed:
existing.cached_models = _json2.dumps(probed)
if mlx_shim_model_id:
existing.cached_models = json.dumps([mlx_shim_model_id])
existing.pinned_models = json.dumps([mlx_shim_model_id])
db.commit()
else:
from routes.model_routes import _probe_endpoint
import json as _json2
probed = _probe_endpoint(base_url, existing.api_key, timeout=5)
if probed:
existing.cached_models = _json2.dumps(probed)
db.commit()
except Exception as _pe:
logger.warning(f"Re-probe failed for {base_url}: {_pe!r}")
# Sweep stale dupes: other endpoints with the same display name
@@ -1492,13 +1844,19 @@ def setup_cookbook_routes() -> APIRouter:
# to a minute later) and the picker shows nothing — even
# though the endpoint is in the DB and the server is up.
try:
from routes.model_routes import _probe_endpoint
import json as _json2
probed = _probe_endpoint(base_url, None, timeout=5)
if probed:
ep.cached_models = _json2.dumps(probed)
if mlx_shim_model_id:
ep.cached_models = json.dumps([mlx_shim_model_id])
ep.pinned_models = json.dumps([mlx_shim_model_id])
db.commit()
logger.info(f"Auto-register: probed {len(probed)} models @ {base_url}")
logger.info(f"Auto-register: pinned MLX DeepSeek-V4 shim model @ {base_url}")
else:
from routes.model_routes import _probe_endpoint
import json as _json2
probed = _probe_endpoint(base_url, None, timeout=5)
if probed:
ep.cached_models = _json2.dumps(probed)
db.commit()
logger.info(f"Auto-register: probed {len(probed)} models @ {base_url}")
except Exception as _pe:
logger.warning(f"Auto-register: probe-after-create failed for {base_url}: {_pe!r}")
return ep_id
@@ -1541,6 +1899,7 @@ def setup_cookbook_routes() -> APIRouter:
req.cmd = _validate_serve_cmd(req.cmd) or ""
req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or ""
req.cmd = _normalize_minimax_m3_vllm_cmd(req.cmd)
req.cmd = _normalize_deepseek_v4_sglang_cmd(req.cmd)
req.cmd = _venv_safe_local_pip_install_cmd(
req.cmd,
local=not bool(req.remote_host),
@@ -1719,6 +2078,9 @@ def setup_cookbook_routes() -> APIRouter:
runner_lines.append(_safe_env_prefix(req.env_prefix))
else:
runner_lines.append("deactivate 2>/dev/null; hash -r")
_append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd)
if "sglang.launch_server" in req.cmd or "mlx_lm.server" in req.cmd or re.search(r"\bvllm\s+serve\b", req.cmd or ""):
_append_openai_port_preflight_lines(runner_lines, cmd=req.cmd, expected_model=req.repo_id)
# Show whether the HF token reached this server (masked) — a gated
# model vLLM has to download will be denied without it.
runner_lines.append(_HF_TOKEN_STATUS_SNIPPET)
@@ -1990,6 +2352,116 @@ def setup_cookbook_routes() -> APIRouter:
runner_lines.append(' printf "%s\\n" "$ODYSSEUS_SGLANG_IMPORT_ERROR"')
runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127')
runner_lines.append('fi')
elif "mlx_lm.server" in req.cmd:
runner_lines.append('export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
runner_lines.append('if ! ODYSSEUS_MLX_IMPORT_ERROR="$(python3 -c "import mlx_lm" 2>&1)"; then')
runner_lines.append(' echo "ERROR: MLX LM is not installed."')
runner_lines.append(' printf "%s\\n" "$ODYSSEUS_MLX_IMPORT_ERROR"')
runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127')
runner_lines.append('fi')
runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then')
runner_lines.append(' ODYSSEUS_MLX_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
runner_lines.append('import shlex, sys')
runner_lines.append('parts = shlex.split(sys.argv[1])')
runner_lines.append('py = "python3"')
runner_lines.append('for i, part in enumerate(parts):')
runner_lines.append(' if part.endswith("/bin/python") or part.endswith("/bin/python3") or "/bin/python3." in part:')
runner_lines.append(' py = part')
runner_lines.append(' break')
runner_lines.append('print(py)')
runner_lines.append('PY')
runner_lines.append(')"')
runner_lines.append(' ODYSSEUS_SERVE_CMD="$("$ODYSSEUS_MLX_CMD_PY" - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
runner_lines.append('import json, os, shlex, sys')
runner_lines.append('from pathlib import Path')
runner_lines.append('parts = shlex.split(sys.argv[1])')
runner_lines.append('try:')
runner_lines.append(' i = parts.index("--model")')
runner_lines.append(' model = parts[i + 1]')
runner_lines.append('except Exception:')
runner_lines.append(' print(shlex.join(parts)); raise SystemExit')
runner_lines.append('if "/" in model and not model.startswith("/") and model.startswith("mlx-community/"):')
runner_lines.append(' roots = []')
runner_lines.append(' def add(p):')
runner_lines.append(' if not p: return')
runner_lines.append(' p = os.path.expanduser(p)')
runner_lines.append(' if p not in roots: roots.append(p)')
runner_lines.append(' add(os.environ.get("HUGGINGFACE_HUB_CACHE"))')
runner_lines.append(' hf_home = os.environ.get("HF_HOME")')
runner_lines.append(' if hf_home: add(os.path.join(hf_home, "hub"))')
runner_lines.append(' add("~/.cache/huggingface/hub")')
runner_lines.append(' cache_name = "models--" + model.replace("/", "--")')
runner_lines.append(' best = ""')
runner_lines.append(' best_mtime = -1.0')
runner_lines.append(' for root in roots:')
runner_lines.append(' snap_root = os.path.join(root, cache_name, "snapshots")')
runner_lines.append(' if not os.path.isdir(snap_root): continue')
runner_lines.append(' for name in os.listdir(snap_root):')
runner_lines.append(' path = os.path.join(snap_root, name)')
runner_lines.append(' if not os.path.isdir(path): continue')
runner_lines.append(' if not os.path.exists(os.path.join(path, "config.json")): continue')
runner_lines.append(' try: mtime = os.path.getmtime(path)')
runner_lines.append(' except OSError: mtime = 0')
runner_lines.append(' if mtime > best_mtime:')
runner_lines.append(' best, best_mtime = path, mtime')
runner_lines.append(' if best:')
runner_lines.append(' print("[odysseus] MLX using cached snapshot:", best, file=sys.stderr)')
runner_lines.append(' launch_model = best')
runner_lines.append(' if "deepseek-v4" in model.lower():')
runner_lines.append(' try:')
runner_lines.append(' import mlx_lm.models.deepseek_v4 as dsv4')
runner_lines.append(' import mlx_lm.utils as mlx_utils')
runner_lines.append(' utils_path = Path(mlx_utils.__file__)')
runner_lines.append(' utils_text = utils_path.read_text()')
runner_lines.append(' utils_needle = \' def class_predicate(p, m):\\n # Handle custom per layer quantizations\\n if p in config["quantization"]:\\n return config["quantization"][p]\\n if not hasattr(m, "to_quantized"):\\n return False\\n return f"{p}.scales" in weights\\n\'')
runner_lines.append(' utils_repl = \' def class_predicate(p, m):\\n # Odysseus: DeepSeek-V4 MXFP4 switch layers may already be quantized.\\n if type(m).__name__ == "QuantizedSwitchLinear":\\n return False\\n # Handle custom per layer quantizations\\n if p in config["quantization"]:\\n return config["quantization"][p]\\n if not hasattr(m, "to_quantized"):\\n return False\\n return f"{p}.scales" in weights\\n\'')
runner_lines.append(' if utils_repl not in utils_text and utils_needle in utils_text:')
runner_lines.append(' bak = utils_path.with_suffix(utils_path.suffix + ".odysseus_bak")')
runner_lines.append(' if not bak.exists(): bak.write_text(utils_text)')
runner_lines.append(' utils_path.write_text(utils_text.replace(utils_needle, utils_repl))')
runner_lines.append(' print("[odysseus] Patched MLX-LM QuantizedSwitchLinear double-quantization guard.", file=sys.stderr)')
runner_lines.append(' dsv4_path = Path(dsv4.__file__)')
runner_lines.append(' dsv4_text = dsv4_path.read_text()')
runner_lines.append(' dsv4_needle = \' for sub in ("attn", "ffn"):\\n for p in ("fn", "base", "scale"):\\n nk = nk.replace(f".hc_{sub}_{p}", f".hc_{sub}.{p}")\\n for wo, wn in w_remap.items():\\n\'')
runner_lines.append(' dsv4_repl = \' for sub in ("attn", "ffn"):\\n for p in ("fn", "base", "scale"):\\n nk = nk.replace(f".hc_{sub}_{p}", f".hc_{sub}.{p}")\\n # Odysseus: normalize alternate hyper-connection key aliases.\\n nk = nk.replace(".attn_hc.", ".hc_attn.")\\n nk = nk.replace(".ffn_hc.", ".hc_ffn.")\\n for wo, wn in w_remap.items():\\n\'')
runner_lines.append(' if dsv4_repl not in dsv4_text and dsv4_needle in dsv4_text:')
runner_lines.append(' bak = dsv4_path.with_suffix(dsv4_path.suffix + ".odysseus_bak")')
runner_lines.append(' if not bak.exists(): bak.write_text(dsv4_text)')
runner_lines.append(' dsv4_path.write_text(dsv4_text.replace(dsv4_needle, dsv4_repl))')
runner_lines.append(' print("[odysseus] Patched MLX-LM DeepSeek-V4 hyper-connection key aliases.", file=sys.stderr)')
runner_lines.append(' except Exception as e:')
runner_lines.append(' print("[odysseus] WARNING: failed to apply MLX DeepSeek-V4 compatibility patch:", e, file=sys.stderr)')
runner_lines.append(' try:')
runner_lines.append(' src = Path(best)')
runner_lines.append(' shim = Path.home() / ".cache" / "odysseus" / "mlx-shims" / src.name')
runner_lines.append(' if len(src.name) > 20:')
runner_lines.append(' shim = Path.home() / ".cache" / "odysseus" / "mlx-shims" / model.split("/")[-1]')
runner_lines.append(' shim.mkdir(parents=True, exist_ok=True)')
runner_lines.append(' for child in src.iterdir():')
runner_lines.append(' target = shim / child.name')
runner_lines.append(' if child.name == "tokenizer_config.json":')
runner_lines.append(' continue')
runner_lines.append(' if target.exists() or target.is_symlink():')
runner_lines.append(' continue')
runner_lines.append(' target.symlink_to(child)')
runner_lines.append(' tc_path = src / "tokenizer_config.json"')
runner_lines.append(' if tc_path.exists():')
runner_lines.append(' tc = json.loads(tc_path.read_text())')
runner_lines.append(' if tc.get("tool_parser_type") == "deepseek_v4":')
runner_lines.append(' tc.pop("tool_parser_type", None)')
runner_lines.append(' (shim / "tokenizer_config.json").write_text(json.dumps(tc, indent=2, ensure_ascii=False))')
runner_lines.append(' launch_model = str(shim)')
runner_lines.append(' print("[odysseus] MLX DeepSeek-V4 using sanitized shim:", launch_model, file=sys.stderr)')
runner_lines.append(' except Exception as e:')
runner_lines.append(' print("[odysseus] WARNING: failed to create MLX DeepSeek-V4 shim:", e, file=sys.stderr)')
runner_lines.append(' parts[i + 1] = launch_model')
runner_lines.append(' else:')
runner_lines.append(' print("[odysseus] MLX cached snapshot not found for:", model, file=sys.stderr)')
runner_lines.append('print(shlex.join(parts))')
runner_lines.append('PY')
runner_lines.append(')"')
runner_lines.append('fi')
elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
runner_lines.append('if ! ODYSSEUS_DIFFUSION_IMPORT_ERROR="$(python3 -c "import torch, diffusers" 2>&1)"; then')
@@ -2021,8 +2493,12 @@ def setup_cookbook_routes() -> APIRouter:
runner_lines,
keep_shell_open=not local_windows,
)
if "vllm serve" in req.cmd:
if "vllm serve" in req.cmd or "mlx_lm.server" in req.cmd:
runner_lines.append('eval "$ODYSSEUS_SERVE_CMD"')
elif is_pip_install:
if not is_windows and (req.platform or "").lower() in {"darwin", "macos"}:
req.cmd = _pip_install_command_without_break_system_packages(req.cmd)
_append_pip_install_runner_lines(runner_lines, req.cmd)
else:
runner_lines.append(req.cmd)
if local_windows:
@@ -2071,7 +2547,7 @@ def setup_cookbook_routes() -> APIRouter:
setup_cmd = (
f"{scp_extras}"
f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && "
f"ssh {_pf}{remote} 'chmod +x {remote_runner} && tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} \"./{remote_runner}\"'"
f"ssh {_pf}{remote} {shlex.quote(_remote_tmux_launch_command(session_id, remote_runner))}"
)
else:
setup_cmd = f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}"
@@ -2400,6 +2876,59 @@ def setup_cookbook_routes() -> APIRouter:
gpus[0]["busy"] = True
return gpus
async def _probe_apple_unified_memory(host: str | None, ssh_port: str | None) -> dict | None:
"""Best-effort Apple Silicon unified-memory probe, local or over SSH."""
cmd = (
"uname -s; uname -m; "
"sysctl -n hw.memsize 2>/dev/null || true; "
"vm_stat 2>/dev/null | awk '"
"/page size of/ {gsub(/[^0-9]/, \"\", $8); page=$8} "
"/Pages free/ {gsub(/[^0-9]/, \"\", $3); free=$3} "
"/Pages inactive/ {gsub(/[^0-9]/, \"\", $3); inactive=$3} "
"/Pages speculative/ {gsub(/[^0-9]/, \"\", $3); speculative=$3} "
"/Pages purgeable/ {gsub(/[^0-9]/, \"\", $3); purgeable=$3} "
"END {if (!page) page=16384; print page, free+inactive+speculative+purgeable}'"
)
out, err = await _run_gpu_shell(cmd, host, ssh_port, timeout=6)
if err is not None or not out:
return None
lines = [ln.strip() for ln in out.splitlines() if ln.strip()]
if len(lines) < 4:
return None
if lines[0] != "Darwin" or lines[1] not in {"arm64", "arm64e"}:
return None
try:
total_bytes = int(lines[2])
page_parts = lines[3].split()
page_size = int(page_parts[0])
available_pages = int(page_parts[1])
except (ValueError, IndexError):
return None
if total_bytes <= 0:
return None
total_mb = int(total_bytes / (1024 * 1024))
free_mb = max(0, min(total_mb, int((page_size * available_pages) / (1024 * 1024))))
used_mb = max(0, total_mb - free_mb)
return {
"ok": True,
"gpus": [{
"index": 0,
"name": "Apple Silicon unified memory",
"uuid": "apple-metal-0",
"free_mb": free_mb,
"total_mb": total_mb,
"used_mb": used_mb,
"util_pct": 0,
"busy": bool(total_mb and (free_mb / total_mb) < 0.2),
"processes": [],
"backend": "metal",
"source": "apple-vm-stat",
"unified_memory": True,
}],
"backend": "metal",
"source": "apple-vm-stat",
}
@router.get("/api/cookbook/gpus")
async def list_gpus(request: Request, host: str | None = None, ssh_port: str | None = None):
"""Probe GPU memory/process state locally or via SSH.
@@ -2529,6 +3058,12 @@ def setup_cookbook_routes() -> APIRouter:
except Exception as e:
logger.warning("Apple Metal GPU fallback failed: %s", e)
apple_gpus = await _probe_apple_unified_memory(host, ssh_port)
if apple_gpus:
apple_gpus["fallback_from"] = "nvidia-smi"
apple_gpus["nvidia_error"] = nvidia_error
return apple_gpus
amd_gpus = await _probe_amd_sysfs(host, ssh_port)
if amd_gpus:
# The per-GPU dict already carries the runtime label picked by
@@ -2974,7 +3509,7 @@ def setup_cookbook_routes() -> APIRouter:
try:
ls = subprocess.run(
ssh_base + [host, "tmux ls 2>/dev/null"],
ssh_base + [host, _remote_tmux_command("ls")],
timeout=6, capture_output=True, text=True,
)
except Exception:
@@ -2987,7 +3522,7 @@ def setup_cookbook_routes() -> APIRouter:
continue
try:
cap = subprocess.run(
ssh_base + [host, "tmux", "capture-pane", "-t", sid, "-p", "-S", "-300"],
ssh_base + [host, _remote_tmux_command("capture-pane", "-t", sid, "-p", "-S", "-300")],
timeout=6, capture_output=True, text=True,
)
pane = cap.stdout or ""
@@ -3009,16 +3544,27 @@ def setup_cookbook_routes() -> APIRouter:
m_repo = re.search(r"snapshot_download\(\s*repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text)
if not m_repo:
m_repo = re.search(r"(?:https://huggingface\.co/)?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", script_text)
if not m_repo and pane:
m_cache = re.search(r"models--([A-Za-z0-9_.-]+)--([A-Za-z0-9_.-]+)", pane)
if m_cache:
repo_id = f"{m_cache.group(1)}/{m_cache.group(2)}"
repo_id = m_repo.group(1) if m_repo else f"adopted:{sid}"
adopted_download_done = "DOWNLOAD_OK" in pane
if repo_id.startswith("adopted:") and pane:
m_cache = re.search(r"models--([A-Za-z0-9_.-]+)--([A-Za-z0-9_.-]+)", pane)
if m_cache:
repo_id = f"{m_cache.group(1)}/{m_cache.group(2)}"
import time as _t2
tasks.append({
"id": sid,
"sessionId": sid,
"name": repo_id.split("/")[-1] if "/" in repo_id else repo_id,
"type": "download",
"status": "running",
"status": "completed" if adopted_download_done else "running",
"progress": "Download complete" if adopted_download_done else "",
"output": (pane or f"Auto-adopted from orphan tmux download session on {host}.")[-5000:],
"ts": int(_t2.time() * 1000),
"completedAt": int(_t2.time() * 1000) if adopted_download_done else None,
"payload": {
"repo_id": repo_id,
"remote_host": host,
@@ -3027,6 +3573,7 @@ def setup_cookbook_routes() -> APIRouter:
"remoteHost": host,
"sshPort": sport,
"platform": "linux",
"_cacheComplete": adopted_download_done,
"_adoptedExternally": True,
})
known_sids.add(sid)
@@ -3055,7 +3602,7 @@ def setup_cookbook_routes() -> APIRouter:
except Exception:
cur = []
LIVE_PROCS = {"python", "python3", "vllm", "llama-server",
"llama_cpp_main", "sglang", "lmdeploy",
"llama_cpp_main", "sglang", "mlx_lm", "lmdeploy",
"ollama", "node", "uvicorn"}
if not any(c in LIVE_PROCS for c in cur):
continue
@@ -3635,13 +4182,13 @@ def setup_cookbook_routes() -> APIRouter:
ssh_base = ["ssh"]
if _tport and _tport != "22":
ssh_base.extend(["-p", str(_tport)])
check_cmd = ssh_base + [remote, "tmux", "has-session", "-t", session_id]
check_cmd = ssh_base + [remote, _remote_tmux_command("has-session", "-t", session_id)]
# Capture 500 lines (was 50) so a Python traceback survives
# the post-crash neofetch banner + bash prompt that otherwise
# fills the visible tail. Without this, output_tail ends up
# as just "Locale: C / Ubuntu_Odysseus ❯" and the agent
# can't diagnose the actual error.
capture_cmd = ssh_base + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-500"]
capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-500")]
elif IS_WINDOWS:
# LOCAL Windows task: launched as a detached process (no tmux).
# Liveness comes from the <session>.pid file, output from the
@@ -3655,7 +4202,7 @@ def setup_cookbook_routes() -> APIRouter:
local_win_task = (not remote) and IS_WINDOWS
progress_text = ""
full_snapshot = ""
full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else ""
if local_win_task:
# File-based liveness + output for the detached-process model.
@@ -3684,9 +4231,14 @@ def setup_cookbook_routes() -> APIRouter:
# stopped tasks). The agent's `list_served_models` call
# was blocking the chat stream every time.
_task_status = (task.get("status") or "").lower()
_persisted_serve_ready = (
task_type == "serve"
and bool(full_snapshot)
and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready"
)
if _task_status in {"stopped", "done", "completed",
"crashed", "error", "failed",
"ended", "killed"}:
"ended", "killed"} and not _persisted_serve_ready:
is_alive = False
# Keep the persisted output_tail for the UI — it's
# what the agent uses to diagnose past failures.
@@ -3790,7 +4342,7 @@ def setup_cookbook_routes() -> APIRouter:
# Parse structured phase info — single source of truth for the UI
phase_info = _parse_serve_phase(full_snapshot, task_type) if (task_type == "serve" and full_snapshot) else {}
if phase_info.get("status") == "ready":
if phase_info.get("status") == "ready" and is_alive:
status = "ready"
serve_phase = phase_info.get("phase", "")
diagnosis = _diagnose_serve_output(full_snapshot) if task_type == "serve" and full_snapshot else None
routes/document_routes.py
+71 -10
View File
@@ -54,6 +54,18 @@ def _library_language_for_document(doc: Document) -> str:
return doc.language or "text"
def _email_source_key(content: str) -> tuple[str, str]:
"""Return the source email identity embedded in an email draft document."""
import re
text = content or ""
uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text)
folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text)
uid = (uid_m.group(1).strip() if uid_m else "")
folder = (folder_m.group(1).strip() if folder_m else "INBOX")
return uid, folder
from routes.document_helpers import (
DocumentCreate, DocumentUpdate, DocumentPatch,
_doc_to_dict, _version_to_dict,
@@ -99,24 +111,62 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# the existing lenient path.
session = _get_session_or_404(db, req.session_id, user)
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
# If no language was supplied (e.g. cloning a doc whose language
# was never set), detect it from the content rather than storing
# NULL — which made the editor fall back to plain text. Defaults
# to markdown for prose.
language = req.language
if not language:
from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language
from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content
language = _sniff_doc_language(req.content)
else:
from src.agent_tools.document_tools import _looks_like_email_document
from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content
if _looks_like_email_document(req.content, req.title):
language = "email"
_assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
# Reply drafts are keyed to the source email. If a UI/tool path tries
# to create a second draft for the same email in the same chat,
# update the existing draft instead so quoted thread history stays
# attached to the visible document.
if language == "email" and req.session_id:
source_uid, source_folder = _email_source_key(req.content)
if source_uid:
candidates = (
db.query(Document)
.filter(Document.session_id == req.session_id)
.filter(Document.is_active == True)
.filter(Document.language == "email")
.order_by(Document.updated_at.desc())
.limit(25)
.all()
)
for existing in candidates:
old_uid, old_folder = _email_source_key(existing.current_content or "")
if old_uid != source_uid or old_folder != source_folder:
continue
merged = _coerce_email_document_content(existing.current_content or "", req.content)
if existing.current_content != merged:
new_ver = (existing.version_count or 1) + 1
existing.current_content = merged
existing.title = req.title or existing.title
existing.version_count = new_ver
db.add(DocumentVersion(
id=str(uuid.uuid4()),
document_id=existing.id,
version_number=new_ver,
content=merged,
summary="Updated existing email draft",
source="user",
))
db.commit()
db.refresh(existing)
return _doc_to_dict(existing)
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
doc = Document(
id=doc_id,
session_id=req.session_id,
@@ -570,12 +620,23 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
raise HTTPException(404, "Document not found")
_verify_doc_owner(db, doc, user)
incoming_content = req.content
from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document
is_email_doc = (
(doc.language or "").lower() == "email"
or _looks_like_email_document(doc.current_content or "", doc.title or "")
or _looks_like_email_document(req.content or "", doc.title or "")
)
if is_email_doc:
incoming_content = _coerce_email_document_content(doc.current_content or "", req.content)
doc.language = "email"
# Skip if content is identical unless the caller explicitly wants
# a checkpoint version from the current editor state.
if doc.current_content == req.content and not req.force_version:
if doc.current_content == incoming_content and not req.force_version:
return _doc_to_dict(doc)
_assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
_assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler)
# Check if we can coalesce with the latest version
latest_ver = db.query(DocumentVersion).filter(
@@ -591,7 +652,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
age = (now - ver_time).total_seconds()
if age < VERSION_COALESCE_SECONDS:
# Update the existing version in-place
latest_ver.content = req.content
latest_ver.content = incoming_content
latest_ver.created_at = now
if req.summary:
latest_ver.summary = req.summary
@@ -603,14 +664,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
id=str(uuid.uuid4()),
document_id=doc_id,
version_number=new_ver,
content=req.content,
content=incoming_content,
summary=req.summary or "Manual edit",
source="user",
)
doc.version_count = new_ver
db.add(ver)
doc.current_content = req.content
doc.current_content = incoming_content
db.commit()
db.refresh(doc)
return _doc_to_dict(doc)
routes/email_routes.py
+244 -27
View File
@@ -494,23 +494,37 @@ def _email_index_rows(owner: str, account_id: str | None, folder: str, uids: lis
return out
def _email_index_search(owner: str, account_id: str | None, folder: str, query: str, limit: int) -> tuple[list[dict], int, str | None]:
def _email_index_search(owner: str, account_id: str | None, folder: str, query: str, limit: int, global_search: bool = True) -> tuple[list[dict], int, str | None]:
q = (query or "").strip()
if not q:
return [], 0, None
limit = max(1, min(int(limit or 50), 200))
account_key = _account_cache_key(account_id, owner)
like = "%" + q.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
folder_clause = ""
params: list = [owner or "", account_key]
# Searching from INBOX should feel global for Gmail-style accounts,
# because users expect archived/labelled mail to show up too. The
# local index only contains folders that have been warmed/listed, so
# this remains a best-effort fast path; IMAP is still the fallback.
if (folder or "").upper() != "INBOX":
if not global_search or (folder or "").upper() != "INBOX":
folder_clause = "AND folder=?"
params.append(folder)
params.extend([like, like, like, like, like])
terms = _email_search_terms(q)
if not terms:
return [], 0, None
term_clause = " AND ".join([
"""(
subject LIKE ? ESCAPE '\\' OR
from_name LIKE ? ESCAPE '\\' OR
from_address LIKE ? ESCAPE '\\' OR
to_text LIKE ? ESCAPE '\\' OR
cc_text LIKE ? ESCAPE '\\'
)"""
for _ in terms
])
for term in terms:
like = "%" + term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
params.extend([like, like, like, like, like])
try:
conn = _sql3.connect(SCHEDULED_DB)
try:
@@ -519,13 +533,7 @@ def _email_index_search(owner: str, account_id: str | None, folder: str, query:
SELECT COUNT(*), MAX(updated_at)
FROM email_message_index
WHERE owner=? AND account_key=? {folder_clause}
AND (
subject LIKE ? ESCAPE '\\' OR
from_name LIKE ? ESCAPE '\\' OR
from_address LIKE ? ESCAPE '\\' OR
to_text LIKE ? ESCAPE '\\' OR
cc_text LIKE ? ESCAPE '\\'
)
AND {term_clause}
""",
params,
).fetchone()
@@ -539,13 +547,7 @@ def _email_index_search(owner: str, account_id: str | None, folder: str, query:
folder
FROM email_message_index
WHERE owner=? AND account_key=? {folder_clause}
AND (
subject LIKE ? ESCAPE '\\' OR
from_name LIKE ? ESCAPE '\\' OR
from_address LIKE ? ESCAPE '\\' OR
to_text LIKE ? ESCAPE '\\' OR
cc_text LIKE ? ESCAPE '\\'
)
AND {term_clause}
ORDER BY date_epoch DESC
LIMIT ?
""",
@@ -583,6 +585,64 @@ def _email_index_search(owner: str, account_id: str | None, folder: str, query:
return emails, total, (total_row or [None, None])[1]
def _email_search_terms(query: str) -> list[str]:
q = (query or "").strip()
if not q:
return []
# Preserve quoted phrases, then split the rest. This makes:
# honda insurance -> honda AND insurance
# "Yoko Honda" insurance -> "Yoko Honda" AND insurance
# The cap avoids creating huge IMAP expressions from pasted paragraphs.
parts = []
consumed = []
for m in re.finditer(r'"([^"]{1,120})"', q):
phrase = m.group(1).strip()
if phrase:
parts.append(phrase)
consumed.append((m.start(), m.end()))
remainder = q
for start, end in reversed(consumed):
remainder = remainder[:start] + " " + remainder[end:]
parts.extend(re.findall(r"[^\s,;]+", remainder))
out = []
seen = set()
for p in parts:
p = p.strip().strip('"').strip()
if len(p) < 2:
continue
key = p.lower()
if key in seen:
continue
seen.add(key)
out.append(p)
if len(out) >= 6:
break
return out
def _imap_or_many(keys: list[str]) -> str:
if not keys:
return "ALL"
expr = keys[0]
for key in keys[1:]:
expr = f"OR ({expr}) ({key})"
return expr
def _email_imap_search_criteria(query: str) -> str:
terms = _email_search_terms(query)
if not terms:
return "ALL"
term_exprs = []
for term in terms:
q = _imap_search_quote(term)
# Search both sides of the conversation, plus subject and body. The
# older route only searched FROM/SUBJECT/TEXT, so recipient searches
# and many sent-message searches felt broken.
term_exprs.append(f"({_imap_or_many([f'FROM {q}', f'TO {q}', f'CC {q}', f'SUBJECT {q}', f'TEXT {q}'])})")
return "(" + " ".join(term_exprs) + ")"
def _email_index_upsert(owner: str, account_id: str | None, folder: str, emails: list[dict]):
if not emails:
return
@@ -2059,6 +2119,7 @@ def setup_email_routes():
limit: int = Query(50),
account_id: str | None = Query(None),
local_only: bool = Query(False),
scope: str = Query("all"),
owner: str = Depends(require_owner),
):
"""Search emails server-side via IMAP SEARCH. Matches subject, from, or body text.
@@ -2072,9 +2133,10 @@ def setup_email_routes():
# CRLF in q would terminate the IMAP command early — reject defensively.
if "\r" in q or "\n" in q:
raise HTTPException(400, "Invalid query")
global_search = (scope or "all").lower() != "folder"
indexed_response = None
try:
indexed_emails, indexed_total, indexed_at = _email_index_search(owner, account_id, folder, q, limit)
indexed_emails, indexed_total, indexed_at = _email_index_search(owner, account_id, folder, q, limit, global_search=global_search)
indexed_response = {
"emails": indexed_emails,
"total": indexed_total,
@@ -2093,7 +2155,7 @@ def setup_email_routes():
# If the user asked for INBOX, try to upgrade to All Mail —
# one folder == every email on Gmail-class servers.
effective_folder = folder
if (folder or "").upper() == "INBOX":
if global_search and (folder or "").upper() == "INBOX":
try:
status, folder_lines = conn.list()
if status == "OK" and folder_lines:
@@ -2112,12 +2174,18 @@ def setup_email_routes():
pass
conn.select(_q(effective_folder), readonly=True)
# Escape backslash and quote for the IMAP-SEARCH quoted-string.
q_escaped = q.replace('\\', '\\\\').replace('"', '\\"')
search_cmd = f'(OR OR FROM "{q_escaped}" SUBJECT "{q_escaped}" TEXT "{q_escaped}")'
search_cmd = _email_imap_search_criteria(q)
status, data = _imap_uid_search(conn, search_cmd)
if status != "OK" or not data[0]:
if indexed_response and indexed_response.get("emails"):
indexed_response["fallback"] = True
indexed_response["source"] = "index"
indexed_response["sync"] = {
**(indexed_response.get("sync") or {}),
"fallback_reason": "imap_empty" if status == "OK" else "imap_search_failed",
}
return indexed_response
return {
"emails": [],
"total": 0,
@@ -3213,6 +3281,155 @@ def setup_email_routes():
logger.error(f"Failed to upload attachment: {e}")
return {"success": False, "error": "Mail operation failed"}
def _safe_compose_filename(name: str, fallback: str = "attachment") -> str:
safe_name = re.sub(r"[^\w\s\-.]", "_", Path(str(name or fallback)).name).strip(". ")[:180]
return safe_name or fallback
def _stage_compose_bytes(filename: str, content: bytes) -> dict:
if len(content) > EMAIL_COMPOSE_UPLOAD_MAX_BYTES:
raise HTTPException(status_code=413, detail="Attachment too large")
safe_name = _safe_compose_filename(filename)
token = f"{uuid.uuid4().hex}_{safe_name}"
filepath = COMPOSE_UPLOADS_DIR / token
with open(filepath, "wb") as f:
f.write(content)
return {"success": True, "token": token, "filename": safe_name, "size": len(content)}
def _stage_compose_file(filename: str, src: Path) -> dict:
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="File not found")
size = src.stat().st_size
if size > EMAIL_COMPOSE_UPLOAD_MAX_BYTES:
raise HTTPException(status_code=413, detail="Attachment too large")
safe_name = _safe_compose_filename(filename)
token = f"{uuid.uuid4().hex}_{safe_name}"
dest = COMPOSE_UPLOADS_DIR / token
import shutil as _shutil
_shutil.copyfile(str(src), str(dest))
return {"success": True, "token": token, "filename": safe_name, "size": size}
def _load_odysseus_attachment_source(db, kind: str, item_id: str, owner: str):
from core.database import Document as _Doc, GalleryImage as _GI
from core.database import Session as _Sess
if kind == "document":
doc = db.query(_Doc).filter(_Doc.id == item_id, _Doc.is_active == True).first()
if not doc:
raise HTTPException(status_code=404, detail="Document not found")
if owner:
if doc.owner and doc.owner != owner:
raise HTTPException(status_code=404, detail="Document not found")
if not doc.owner and doc.session_id:
sess = db.query(_Sess).filter(_Sess.id == doc.session_id).first()
if sess and sess.owner and sess.owner != owner:
raise HTTPException(status_code=404, detail="Document not found")
lang = (doc.language or "text").strip().lower()
ext = {
"markdown": "md",
"email": "eml",
"json": "json",
"yaml": "yaml",
"yml": "yml",
"html": "html",
"csv": "csv",
"xml": "xml",
"text": "txt",
}.get(lang, "txt")
base = _safe_compose_filename(doc.title or "document", "document")
if not base.lower().endswith(f".{ext}"):
base = f"{base}.{ext}"
return {"filename": base, "content": (doc.current_content or "").encode("utf-8")}
if kind == "gallery":
img = db.query(_GI).filter(_GI.id == item_id, _GI.is_active == True).first()
if not img:
raise HTTPException(status_code=404, detail="Image not found")
if owner and img.owner and img.owner != owner:
raise HTTPException(status_code=404, detail="Image not found")
from routes.gallery.gallery_routes import _gallery_image_path
src = _gallery_image_path(img.filename)
if not src.exists() or not src.is_file():
raise HTTPException(status_code=404, detail="Image file not found")
return {"filename": _safe_compose_filename(img.filename or "gallery-image.png"), "path": src}
raise HTTPException(status_code=400, detail="Unknown attachment kind")
@router.post("/compose-from-odysseus")
async def compose_from_odysseus(data: dict, owner: str = Depends(require_owner)):
"""Stage an Odysseus document or gallery image as a compose upload."""
kind = str(data.get("kind") or "").strip().lower()
item_id = str(data.get("id") or "").strip()
if kind not in {"document", "gallery"} or not item_id:
raise HTTPException(status_code=400, detail="Expected kind and id")
try:
from core.database import SessionLocal as _SL
db = _SL()
try:
src = _load_odysseus_attachment_source(db, kind, item_id, owner)
if "path" in src:
return _stage_compose_file(src["filename"], src["path"])
return _stage_compose_bytes(src["filename"], src["content"])
finally:
db.close()
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to stage Odysseus attachment {kind}/{item_id}: {e}")
return {"success": False, "error": "Mail operation failed"}
@router.post("/compose-from-odysseus-zip")
async def compose_from_odysseus_zip(data: dict, owner: str = Depends(require_owner)):
"""Stage several Odysseus documents/gallery images as one zip attachment."""
raw_items = data.get("items") or []
if not isinstance(raw_items, list) or not raw_items:
raise HTTPException(status_code=400, detail="Expected items")
if len(raw_items) > 100:
raise HTTPException(status_code=400, detail="Too many attachments")
try:
from core.database import SessionLocal as _SL
db = _SL()
try:
buf = io.BytesIO()
used_names: dict[str, int] = {}
def unique_name(name: str) -> str:
safe = _safe_attachment_zip_name(name, "attachment")
stem = Path(safe).stem or "attachment"
suffix = Path(safe).suffix
idx = used_names.get(safe, 0)
used_names[safe] = idx + 1
if idx == 0:
return safe
return f"{stem}-{idx + 1}{suffix}"
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
for item in raw_items:
if not isinstance(item, dict):
continue
kind = str(item.get("kind") or "").strip().lower()
item_id = str(item.get("id") or "").strip()
if kind not in {"document", "gallery"} or not item_id:
continue
src = _load_odysseus_attachment_source(db, kind, item_id, owner)
zname = unique_name(src["filename"])
if "path" in src:
zf.write(src["path"], arcname=zname)
else:
zf.writestr(zname, src["content"])
content = buf.getvalue()
if not content:
raise HTTPException(status_code=400, detail="No valid attachments")
return _stage_compose_bytes("odysseus-attachments.zip", content)
finally:
db.close()
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to stage Odysseus zip attachment: {e}")
return {"success": False, "error": "Mail operation failed"}
@router.post("/compose-from-attachment/{uid}/{index}")
async def compose_from_attachment(
uid: str,
@@ -4486,7 +4703,9 @@ def setup_email_routes():
cfg["email_auto_tag"] = bool(settings.get("email_auto_tag", False))
cfg["email_auto_spam"] = bool(settings.get("email_auto_spam", False))
cfg["email_auto_calendar"] = bool(settings.get("email_auto_calendar", False))
cfg["email_auto_translate"] = bool(settings.get("email_auto_translate", False))
# Email translation is owned by the background task now; opening an email
# should not trigger reader-side auto-translation from Settings.
cfg["email_auto_translate"] = False
cfg["email_translate_language"] = settings.get("email_translate_language", "English")
return cfg
@@ -4501,11 +4720,9 @@ def setup_email_routes():
"""
# Automation flags stay in settings.json (they're global, not per-account)
settings = _load_settings()
for key in ["email_auto_summarize", "email_auto_reply", "email_auto_tag", "email_auto_spam", "email_auto_calendar", "email_auto_translate"]:
for key in ["email_auto_summarize", "email_auto_reply", "email_auto_tag", "email_auto_spam", "email_auto_calendar"]:
if key in data:
settings[key] = data[key]
if "email_translate_language" in data:
settings["email_translate_language"] = (data.get("email_translate_language") or "English").strip() or "English"
_save_settings(settings)
# Credentials go into the default account row
routes/hwfit_routes.py
+12 -3
View File
@@ -191,7 +191,7 @@ def setup_hwfit_routes():
return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
@router.get("/models")
def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
"""Rank LLM models against detected hardware and return scored results.
gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -200,11 +200,17 @@ def setup_hwfit_routes():
fresh=true bypasses the hardware-detection cache."""
from services.hwfit.hardware import detect_system
from services.hwfit.fit import rank_models
from services.hwfit.models import get_models, model_catalog_path
from services.hwfit.models import get_models, model_catalog_path, refresh_dynamic_catalogs
host, ssh_port = _validate_detection_target(host, ssh_port)
system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
if system.get("error"):
return {"system": system, "models": [], "error": system["error"]}
catalog_refresh = None
if refresh_catalog:
try:
catalog_refresh = refresh_dynamic_catalogs(force=True)
except Exception as e:
catalog_refresh = {"error": str(e)}
if not get_models():
return {
"system": system,
@@ -304,7 +310,10 @@ def setup_hwfit_routes():
rank_kwargs.pop("target_context", None)
rank_kwargs.pop("fit_only", None)
results = rank_models(system, **rank_kwargs)
return {"system": system, "models": results}
payload = {"system": system, "models": results}
if catalog_refresh is not None:
payload["catalog_refresh"] = catalog_refresh
return payload
@router.get("/profiles")
def get_serve_profiles(model: str = "", model_path: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):
routes/model_routes.py
+42 -4
View File
@@ -1152,6 +1152,36 @@ def _merge_model_ids(*lists):
return out
def _is_mlx_deepseek_v4_repo_id(model_id: str) -> bool:
m = str(model_id or "").lower()
return "mlx-community/deepseek-v4" in m
def _is_mlx_deepseek_v4_shim_id(model_id: str) -> bool:
m = str(model_id or "").lower()
return "/.cache/odysseus/mlx-shims/deepseek-v4" in m
def _filter_mlx_deepseek_v4_repo_when_shimmed(model_ids):
"""Hide the broken MLX repo id when a launch-specific shim id is available.
mlx_lm.server may advertise the original HF repo id even though generation
only works through Odysseus' sanitized local shim. Keep the shim as the
submitted model id and remove the raw repo id from the picker/default list.
"""
ids = list(model_ids or [])
has_shim = any(_is_mlx_deepseek_v4_shim_id(m) for m in ids)
if not has_shim:
return ids
return [m for m in ids if not _is_mlx_deepseek_v4_repo_id(m)]
def _model_display_name(model_id: str) -> str:
if _is_mlx_deepseek_v4_shim_id(model_id):
return str(model_id or "").rstrip("/").split("/")[-1] or "DeepSeek-V4-Flash-4bit"
return str(model_id or "").split("/")[-1]
def _visible_models(cached_models, hidden_models, pinned_models=None):
"""Merge cached + pinned model IDs, then filter out hidden ones.
@@ -1165,6 +1195,7 @@ def _visible_models(cached_models, hidden_models, pinned_models=None):
_normalize_model_ids(cached_models),
_normalize_model_ids(pinned_models),
)
merged = _filter_mlx_deepseek_v4_repo_when_shimmed(merged)
if not hidden_models:
return merged
hidden = set(_normalize_model_ids(hidden_models))
@@ -1396,9 +1427,9 @@ def setup_model_routes(model_discovery):
"port": 0,
"url": chat_url,
"models": curated,
"models_display": [mid.split("/")[-1] for mid in curated],
"models_display": [_model_display_name(mid) for mid in curated],
"models_extra": extra,
"models_extra_display": [mid.split("/")[-1] for mid in extra],
"models_extra_display": [_model_display_name(mid) for mid in extra],
"endpoint_id": ep.id,
"endpoint_name": ep.name,
"category": category,
@@ -1426,7 +1457,7 @@ def setup_model_routes(model_discovery):
return {"hosts": [], "items": items}
@router.get("/models")
def api_models(request: Request, refresh: bool = False, background: bool = True):
def api_models(request: Request, refresh: bool = False, background: bool = False):
"""Get available models — per-user (caller sees only their endpoints +
legacy/shared null-owner rows). Cached per-user for 30s."""
# Require auth; "" is the unconfigured single-user mode, treated as
@@ -1887,7 +1918,14 @@ def setup_model_routes(model_discovery):
if api_key.strip() and not existing.api_key:
existing.api_key = api_key.strip()
changed = True
if should_probe:
# Keep duplicate endpoint registration cheap. This path is hit
# by Cookbook/browser auto-register flows and can run while the
# user is sending a chat message. Probing a stale LAN endpoint
# here used to hold the request open for tens of seconds and
# contend with session creation, making "send" feel blocked.
# Explicit "require models" calls still probe; normal refresh
# belongs to /model-endpoints/{id}/models or /probe.
if require_model_list:
probed_models = _probe_endpoint(
base_url,
(api_key.strip() or existing.api_key or None),
routes/session_routes.py
+3 -2
View File
@@ -207,6 +207,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
"""Setup session routes with the provided manager and config"""
REQUEST_TIMEOUT = config.get("REQUEST_TIMEOUT", 20)
SESSION_MODEL_VALIDATION_TIMEOUT = min(float(REQUEST_TIMEOUT or 20), 3.0)
OPENAI_API_KEY = config.get("OPENAI_API_KEY")
SESSIONS_FILE = config.get("SESSIONS_FILE")
@@ -374,7 +375,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
from src.llm_core import list_model_ids
ids = list_model_ids(
endpoint_url,
timeout=REQUEST_TIMEOUT,
timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
headers=validation_headers,
owner=user,
endpoint_id=endpoint_id.strip() if endpoint_id else None,
@@ -394,7 +395,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
req_base = _os.path.basename(model_to_use.rstrip("/"))
avail = list_model_ids(
endpoint_url,
timeout=REQUEST_TIMEOUT,
timeout=SESSION_MODEL_VALIDATION_TIMEOUT,
headers=validation_headers,
owner=user,
endpoint_id=endpoint_id.strip() if endpoint_id else None,
routes/shell_routes.py
+101 -13
View File
@@ -164,6 +164,8 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
return bool(binaries.get("llama-server") or dists.get("llama-cpp-python"))
if name == "sglang":
return bool(dists.get("sglang") or modules.get("sglang", {}).get("real_module"))
if name == "mlx_lm":
return bool(dists.get("mlx-lm") or modules.get("mlx_lm", {}).get("real_module"))
if name == "diffusers":
return bool(
(dists.get("diffusers") or modules.get("diffusers", {}).get("real_module"))
@@ -210,6 +212,10 @@ def _package_status_note(name: str, probe: dict) -> str:
if _package_installed_from_probe(name, probe):
return f"diffusers {dists.get('diffusers', 'available')} with torch {dists.get('torch', 'available')}"
return "Diffusers serving needs both diffusers and torch."
if name == "mlx_lm":
if _package_installed_from_probe(name, probe):
return f"MLX LM {dists.get('mlx-lm', 'available')}"
return "MLX serving needs mlx-lm on an Apple Silicon Mac."
if name in dists:
return f"{name} {dists[name]}"
return ""
@@ -307,12 +313,14 @@ dist_names={{
'vllm':['vllm'],
'llama_cpp':['llama-cpp-python'],
'sglang':['sglang'],
'mlx_lm':['mlx-lm'],
'diffusers':['diffusers','torch'],
'hf_transfer':['hf-transfer','hf_transfer'],
}}
bin_names={{
'vllm':['vllm'],
'llama_cpp':['llama-server'],
'tmux':['tmux'],
}}
def add_user_install_bins_to_path():
@@ -325,6 +333,8 @@ def add_user_install_bins_to_path():
candidates.append(os.path.expanduser('~/llama.cpp/build/bin'))
candidates.append(os.path.expanduser('~/llama.cpp/build-vulkan/bin'))
candidates.append(os.path.expanduser('~/.local/bin'))
candidates.append('/opt/homebrew/bin')
candidates.append('/usr/local/bin')
parts = os.environ.get('PATH', '').split(os.pathsep) if os.environ.get('PATH') else []
changed = False
for path in reversed([p for p in candidates if p]):
@@ -399,6 +409,47 @@ class ShellExecRequest(BaseModel):
use_tmux: bool = False # run in tmux session (survives browser disconnect)
_REMOTE_TMUX_PATH_PREFIX = 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; '
def _normalize_legacy_remote_tmux_exec(command: str) -> str:
"""Repair stale frontend Cookbook tmux SSH commands.
Older loaded JS sends `ssh host 'tmux capture-pane ...'`. On macOS/Homebrew
remotes, non-login SSH shells often lack /opt/homebrew/bin, so tmux is
installed but the capture/kill command returns nothing. Keep this narrowly
scoped to SSH commands whose remote shell starts with `tmux `.
"""
cmd = command or ""
if _REMOTE_TMUX_PATH_PREFIX in cmd or not cmd.lstrip().startswith("ssh "):
return cmd
try:
parts = shlex.split(cmd)
except Exception:
return cmd
if not parts or parts[0] != "ssh":
return cmd
remote_idx = -1
i = 1
while i < len(parts):
part = parts[i]
if part in {"-p", "-o", "-i", "-F", "-J", "-l", "-S", "-W", "-b", "-c", "-m"}:
i += 2
continue
if part.startswith("-"):
i += 1
continue
remote_idx = i
break
if remote_idx < 0 or remote_idx + 1 >= len(parts):
return cmd
remote_cmd = " ".join(parts[remote_idx + 1:]).strip()
if not remote_cmd.startswith("tmux "):
return cmd
repaired = parts[:remote_idx + 1] + [_REMOTE_TMUX_PATH_PREFIX + remote_cmd]
return shlex.join(repaired)
async def _create_shell(command: str, **kwargs):
"""Spawn a shell subprocess for `command`.
@@ -815,6 +866,10 @@ def setup_shell_routes() -> APIRouter:
if not cmd:
return {"stdout": "", "stderr": "No command provided", "exit_code": 1}
fixed_cmd = _normalize_legacy_remote_tmux_exec(cmd)
if fixed_cmd != cmd:
logger.info("Rewrote legacy remote tmux exec command with Homebrew PATH")
cmd = fixed_cmd
logger.info("User shell exec requested: length=%d", len(cmd))
result = await _exec_shell(
cmd, timeout=req.timeout if req.timeout is not None else EXEC_TIMEOUT
@@ -1134,6 +1189,13 @@ def setup_shell_routes() -> APIRouter:
"category": "LLM",
"target": "remote",
},
{
"name": "mlx_lm",
"pip": "mlx-lm",
"desc": "Serve MLX-format models on Apple Silicon Macs",
"category": "LLM",
"target": "remote",
},
{
"name": "APFEL",
"pip": "",
@@ -1279,9 +1341,9 @@ def setup_shell_routes() -> APIRouter:
for name in all_system_names:
qn = shlex.quote(name)
checks.append(
f"if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
f"PATH=\"$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH\"; if command -v {qn} >/dev/null 2>&1; then echo {qn}=1; else echo {qn}=0; fi"
)
checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || true")
checks.append("echo '---OSREL---'; cat /etc/os-release 2>/dev/null || { [ \"$(uname -s 2>/dev/null)\" = \"Darwin\" ] && echo ID=macos; } || true")
inner = " ; ".join(checks)
argv = _ssh_base_argv(host, ssh_port) + [inner]
proc = await asyncio.create_subprocess_exec(
@@ -1538,6 +1600,7 @@ def setup_shell_routes() -> APIRouter:
"onnxruntime",
"hdbscan",
"vllm",
"mlx-lm",
}
if pip_name not in known:
return {"ok": False, "error": f"Unknown package: {pip_name}"}
@@ -1584,6 +1647,19 @@ def setup_shell_routes() -> APIRouter:
elif n == "g++": out += ["gcc-c++"]
else: out.append(n)
return out
def _apk(names):
out = []
for n in names:
if n == "build-essential": out.append("build-base")
else: out.append(n)
return out
def _zypper(names):
out = []
for n in names:
if n == "build-essential": out += ["gcc-c++", "make"]
elif n == "g++": out.append("gcc-c++")
else: out.append(n)
return out
def _brew(names):
return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
# Build a single shell snippet that detects the package manager and
@@ -1592,6 +1668,8 @@ def setup_shell_routes() -> APIRouter:
apt_pkgs = " ".join(shlex.quote(p) for p in _apt(pkgs))
pac_pkgs = " ".join(shlex.quote(p) for p in _pacman(pkgs))
dnf_pkgs = " ".join(shlex.quote(p) for p in _dnf(pkgs))
apk_pkgs = " ".join(shlex.quote(p) for p in _apk(pkgs))
zypper_pkgs = " ".join(shlex.quote(p) for p in _zypper(pkgs))
brew_pkgs = " ".join(shlex.quote(p) for p in _brew(pkgs))
# Error messages go to stderr (>&2) so the route's error field
# gets populated. Without the redirect, `echo "ERROR…"` on stdout
@@ -1599,18 +1677,28 @@ def setup_shell_routes() -> APIRouter:
# bare "HTTP 200" instead of surfacing the real reason.
script = (
'set -e; '
'if ! sudo -n true 2>/dev/null; then '
' echo "ERROR: passwordless sudo unavailable on this target. Run once: sudo apt install -y ' + " ".join(pkgs) + ' (or your distro equivalent: pacman -S, dnf install, brew install). After that, Cookbook can install the rest." >&2; exit 2; fi; '
'if command -v apt-get >/dev/null 2>&1; then '
f' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq && sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
'elif command -v pacman >/dev/null 2>&1; then '
f' sudo -n pacman -Sy --needed --noconfirm {pac_pkgs}; '
'elif command -v dnf >/dev/null 2>&1; then '
f' sudo -n dnf install -y {dnf_pkgs}; '
'elif command -v brew >/dev/null 2>&1; then '
f' brew install {brew_pkgs}; '
'BREW="$(command -v brew 2>/dev/null || true)"; '
'if [ -z "$BREW" ] && [ -x /opt/homebrew/bin/brew ]; then BREW=/opt/homebrew/bin/brew; fi; '
'if [ -z "$BREW" ] && [ -x /usr/local/bin/brew ]; then BREW=/usr/local/bin/brew; fi; '
'if [ -n "$BREW" ]; then '
f' if [ -z "{brew_pkgs}" ]; then echo "Nothing to install with brew for requested packages." >&2; exit 4; fi; "$BREW" install {brew_pkgs}; exit $?; '
'fi; '
'if [ "$(id -u)" = "0" ]; then SUDO=""; '
'elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then SUDO="sudo -n"; '
'else '
' echo "ERROR: no supported package manager (apt/pacman/dnf/brew) on this target." >&2; exit 3; fi'
' echo "ERROR: this target needs sudo for its OS package manager, but passwordless sudo is unavailable. Open a terminal on the target and run the shown install command once, then retry in Cookbook." >&2; exit 2; fi; '
'if command -v apt-get >/dev/null 2>&1; then '
f' $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -qq && $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {apt_pkgs}; '
'elif command -v pacman >/dev/null 2>&1; then '
f' $SUDO pacman -Sy --needed --noconfirm {pac_pkgs}; '
'elif command -v dnf >/dev/null 2>&1; then '
f' $SUDO dnf install -y {dnf_pkgs}; '
'elif command -v apk >/dev/null 2>&1; then '
f' $SUDO apk add --no-interactive {apk_pkgs}; '
'elif command -v zypper >/dev/null 2>&1; then '
f' $SUDO zypper --non-interactive install {zypper_pkgs}; '
'else '
' echo "ERROR: no supported package manager (apt/pacman/dnf/apk/zypper/brew) on this target." >&2; exit 3; fi'
)
try:
if host: