mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-15 12:58:04 +00:00
Merge remote-tracking branch 'origin/dev'
# Conflicts: # routes/document_routes.py
This commit is contained in:
pewdiepie-archdaemon
+211
-54
@@ -32,6 +32,13 @@ from core.platform_compat import (
|
||||
which_tool,
|
||||
)
|
||||
from routes.shell_routes import TMUX_LOG_DIR
|
||||
from src.host_docker_access import (
|
||||
HOST_DOCKER_ACCESS_HINT,
|
||||
HOST_DOCKER_SOCKET_PATH,
|
||||
host_docker_access_enabled,
|
||||
local_docker_available,
|
||||
running_in_container,
|
||||
)
|
||||
from routes.cookbook_output import (
|
||||
error_aware_output_tail, classify_dead_download,
|
||||
HF_CACHE_COMPLETE_PROBE, HF_CACHE_INCOMPLETE_PROBE,
|
||||
@@ -64,6 +71,182 @@ _HF_TOKEN_STATUS_SNIPPET = (
|
||||
'fi'
|
||||
)
|
||||
|
||||
_OLLAMA_SIDECAR_CONTAINERS = {"ollama-test", "ollama-rocm"}
|
||||
_UNSAFE_DOCKER_EXEC_CHARS = frozenset(";&|<>$`\r\n")
|
||||
_SAFE_OLLAMA_MODEL_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$")
|
||||
_SAFE_OLLAMA_FILE_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
|
||||
|
||||
def _is_generated_ollama_docker_exec_cmd(cmd: str | None) -> bool:
|
||||
"""Match only the fixed Docker exec shapes generated by Cookbook."""
|
||||
if not cmd or any(char in cmd for char in _UNSAFE_DOCKER_EXEC_CHARS):
|
||||
return False
|
||||
try:
|
||||
parts = shlex.split(cmd)
|
||||
except ValueError:
|
||||
return False
|
||||
if len(parts) < 4 or parts[:2] != ["docker", "exec"]:
|
||||
return False
|
||||
container, executable = parts[2:4]
|
||||
if container not in _OLLAMA_SIDECAR_CONTAINERS:
|
||||
return False
|
||||
if container == "ollama-rocm" and executable == "ollama":
|
||||
return (
|
||||
len(parts) == 6
|
||||
and parts[4] == "show"
|
||||
and _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(parts[5]) is not None
|
||||
)
|
||||
if container != "ollama-test" or executable != "ollama-import":
|
||||
return False
|
||||
if len(parts) not in {7, 8}:
|
||||
return False
|
||||
model, name, context_size = parts[4:7]
|
||||
return (
|
||||
_SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(model) is not None
|
||||
and _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(name) is not None
|
||||
and re.fullmatch(r"[0-9]+", context_size) is not None
|
||||
and (
|
||||
len(parts) == 7
|
||||
or _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(parts[7]) is not None
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _missing_binary_message(
|
||||
binary: str,
|
||||
target: str,
|
||||
*,
|
||||
local_host_docker_blocked: bool = False,
|
||||
) -> str:
|
||||
if binary == "tmux":
|
||||
return (
|
||||
f"tmux is required for Cookbook background downloads/serves on {target}. "
|
||||
"Install it with your OS package manager, or run Cookbook server setup for that server."
|
||||
)
|
||||
if binary == "docker":
|
||||
if local_host_docker_blocked:
|
||||
return HOST_DOCKER_ACCESS_HINT
|
||||
return (
|
||||
f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. "
|
||||
"Install Docker and make sure this user can run `docker`, then retry."
|
||||
)
|
||||
return f"{binary} is required on {target}, but it was not found."
|
||||
|
||||
|
||||
async def _remote_binary_available(
|
||||
remote: str,
|
||||
ssh_port: str | None,
|
||||
binary: str,
|
||||
*,
|
||||
windows: bool = False,
|
||||
) -> bool:
|
||||
port = ssh_port or ""
|
||||
port_args = ["-p", port] if port and port != "22" else []
|
||||
if windows:
|
||||
check = f'powershell -NoProfile -Command "if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}"'
|
||||
else:
|
||||
check = f"command -v {shlex.quote(binary)} >/dev/null 2>&1"
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ssh",
|
||||
"-o",
|
||||
"ConnectTimeout=6",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
*port_args,
|
||||
remote,
|
||||
check,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await asyncio.wait_for(proc.communicate(), timeout=10)
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def _binary_available(
|
||||
binary: str,
|
||||
remote: str | None,
|
||||
ssh_port: str | None,
|
||||
*,
|
||||
windows: bool = False,
|
||||
in_container: bool | None = None,
|
||||
environ=None,
|
||||
socket_path: str = HOST_DOCKER_SOCKET_PATH,
|
||||
) -> bool:
|
||||
if remote:
|
||||
return await _remote_binary_available(
|
||||
remote,
|
||||
ssh_port,
|
||||
binary,
|
||||
windows=windows,
|
||||
)
|
||||
cli_available = shutil.which(binary) is not None
|
||||
if binary != "docker":
|
||||
return cli_available
|
||||
return local_docker_available(
|
||||
cli_available=cli_available,
|
||||
in_container=in_container,
|
||||
environ=environ,
|
||||
socket_path=socket_path,
|
||||
)
|
||||
|
||||
|
||||
|
||||
def _local_ollama_docker_fallback_available(
|
||||
*,
|
||||
in_container: bool | None = None,
|
||||
environ: dict[str, str] | None = None,
|
||||
socket_path: str = HOST_DOCKER_SOCKET_PATH,
|
||||
) -> bool:
|
||||
return local_docker_available(
|
||||
cli_available=shutil.which("docker") is not None,
|
||||
in_container=in_container,
|
||||
environ=environ,
|
||||
socket_path=socket_path,
|
||||
)
|
||||
|
||||
|
||||
def _local_ollama_docker_access_blocked(
|
||||
*,
|
||||
in_container: bool | None = None,
|
||||
environ: dict[str, str] | None = None,
|
||||
socket_path: str = HOST_DOCKER_SOCKET_PATH,
|
||||
) -> bool:
|
||||
containerized = running_in_container() if in_container is None else in_container
|
||||
if not containerized or shutil.which("docker") is None:
|
||||
return False
|
||||
return not _local_ollama_docker_fallback_available(
|
||||
in_container=containerized,
|
||||
environ=environ,
|
||||
socket_path=socket_path,
|
||||
)
|
||||
|
||||
|
||||
def _append_local_ollama_download_command_lines(
|
||||
lines: list[str],
|
||||
ollama_cmd: str,
|
||||
*,
|
||||
docker_fallback_available: bool,
|
||||
docker_fallback_blocked: bool,
|
||||
) -> None:
|
||||
lines.append('if command -v ollama >/dev/null 2>&1; then')
|
||||
lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}')
|
||||
if docker_fallback_available:
|
||||
lines.append('elif command -v docker >/dev/null 2>&1; then')
|
||||
lines.append(" ODYSSEUS_OLLAMA_CONTAINER=\"$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(ollama-rocm|ollama-test)$' | head -1)\"")
|
||||
lines.append(' if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then')
|
||||
lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}')
|
||||
lines.append(' fi')
|
||||
elif docker_fallback_blocked:
|
||||
hint = shlex.quote("ERROR: " + HOST_DOCKER_ACCESS_HINT)
|
||||
lines.append('else')
|
||||
lines.append(f" printf '%s\\n' {hint}; exit 127")
|
||||
lines.append('fi')
|
||||
lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')
|
||||
|
||||
|
||||
def setup_cookbook_routes() -> APIRouter:
|
||||
router = APIRouter(tags=["cookbook"])
|
||||
_cookbook_state_path = Path(COOKBOOK_STATE_FILE)
|
||||
@@ -445,43 +628,6 @@ def setup_cookbook_routes() -> APIRouter:
|
||||
def _needs_binary(cmd: str, binary: str) -> bool:
|
||||
return bool(re.search(rf"(^|[\s;&|()]){re.escape(binary)}($|[\s;&|()])", cmd or ""))
|
||||
|
||||
def _missing_binary_message(binary: str, target: str) -> str:
|
||||
if binary == "tmux":
|
||||
return (
|
||||
f"tmux is required for Cookbook background downloads/serves on {target}. "
|
||||
"Install it with your OS package manager, or run Cookbook server setup for that server."
|
||||
)
|
||||
if binary == "docker":
|
||||
return (
|
||||
f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. "
|
||||
"Install Docker and make sure this user can run `docker`, then retry."
|
||||
)
|
||||
return f"{binary} is required on {target}, but it was not found."
|
||||
|
||||
async def _remote_binary_available(remote: str, ssh_port: str | None, binary: str, *, windows: bool = False) -> bool:
|
||||
_port = ssh_port or ""
|
||||
_pf = ["-p", _port] if _port and _port != "22" else []
|
||||
if windows:
|
||||
check = f"powershell -NoProfile -Command \"if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}\""
|
||||
else:
|
||||
check = f"command -v {shlex.quote(binary)} >/dev/null 2>&1"
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ssh", "-o", "ConnectTimeout=6", "-o", "StrictHostKeyChecking=no",
|
||||
*_pf, remote, check,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await asyncio.wait_for(proc.communicate(), timeout=10)
|
||||
return proc.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def _binary_available(binary: str, remote: str | None, ssh_port: str | None, *, windows: bool = False) -> bool:
|
||||
if remote:
|
||||
return await _remote_binary_available(remote, ssh_port, binary, windows=windows)
|
||||
return shutil.which(binary) is not None
|
||||
|
||||
def _launch_local_detached(session_id: str, bash_lines: list[str]) -> dict:
|
||||
"""Windows-native stand-in for a LOCAL tmux session (tmux doesn't exist
|
||||
on Windows). Mirrors shell_routes._generate_win_detached / bg_jobs.launch:
|
||||
@@ -610,15 +756,12 @@ def setup_cookbook_routes() -> APIRouter:
|
||||
# slower-but-reliable downloader (resumes cleanly from the .incomplete files).
|
||||
# Use `python3 -m pip` not `pip` — macOS has no bare `pip` command.
|
||||
if is_ollama_download:
|
||||
lines.append('if command -v ollama >/dev/null 2>&1; then')
|
||||
lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}')
|
||||
lines.append('elif command -v docker >/dev/null 2>&1; then')
|
||||
lines.append(' ODYSSEUS_OLLAMA_CONTAINER="$(docker ps --format \'{{.Names}}\' 2>/dev/null | grep -E \'^(ollama-rocm|ollama-test)$\' | head -1)"')
|
||||
lines.append(' if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then')
|
||||
lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}')
|
||||
lines.append(' fi')
|
||||
lines.append('fi')
|
||||
lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')
|
||||
_append_local_ollama_download_command_lines(
|
||||
lines,
|
||||
ollama_cmd,
|
||||
docker_fallback_available=_local_ollama_docker_fallback_available(),
|
||||
docker_fallback_blocked=_local_ollama_docker_access_blocked(),
|
||||
)
|
||||
else:
|
||||
lines.append(f"command -v hf >/dev/null 2>&1 || {_pip_install_fallback_chain('huggingface_hub', upgrade=True)}")
|
||||
if req.disable_hf_transfer:
|
||||
@@ -1384,13 +1527,18 @@ def setup_cookbook_routes() -> APIRouter:
|
||||
req.gpus = _validate_gpus(req.gpus)
|
||||
req.hf_token = req.hf_token or _load_stored_hf_token()
|
||||
_validate_token(req.hf_token)
|
||||
# Normalize away backslash-newline continuations (multi-line pasted
|
||||
# serve commands) so the cleaned single-line command is what gets
|
||||
# written into the runner script and used for engine auto-detection.
|
||||
# `_validate_serve_cmd` returns None for empty input; coerce to "" so the
|
||||
# many downstream `"engine" in req.cmd` membership checks can't hit
|
||||
# `TypeError: argument of type 'NoneType'` (a 500 instead of a clean 400).
|
||||
req.cmd = _validate_serve_cmd(req.cmd) or ""
|
||||
# Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
|
||||
# Keep Docker out of the general allowlist: only these parsed shapes may
|
||||
# proceed to the target-aware Docker availability/opt-in preflight.
|
||||
if _is_generated_ollama_docker_exec_cmd(req.cmd):
|
||||
req.cmd = req.cmd.strip()
|
||||
else:
|
||||
# Normalize away backslash-newline continuations (multi-line pasted
|
||||
# serve commands) so the cleaned single-line command is what gets
|
||||
# written into the runner script and used for engine auto-detection.
|
||||
# `_validate_serve_cmd` returns None for empty input; coerce to "" so
|
||||
# downstream `"engine" in req.cmd` checks cannot raise TypeError.
|
||||
req.cmd = _validate_serve_cmd(req.cmd) or ""
|
||||
req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or ""
|
||||
req.cmd = _normalize_minimax_m3_vllm_cmd(req.cmd)
|
||||
req.cmd = _venv_safe_local_pip_install_cmd(
|
||||
@@ -1468,9 +1616,18 @@ def setup_cookbook_routes() -> APIRouter:
|
||||
"session_id": session_id,
|
||||
}
|
||||
if _needs_binary(req.cmd, "docker") and not await _binary_available("docker", remote, req.ssh_port, windows=is_windows):
|
||||
local_host_docker_blocked = (
|
||||
not remote
|
||||
and running_in_container()
|
||||
and not host_docker_access_enabled()
|
||||
)
|
||||
return {
|
||||
"ok": False,
|
||||
"error": _missing_binary_message("docker", remote or "local server"),
|
||||
"error": _missing_binary_message(
|
||||
"docker",
|
||||
remote or "local server",
|
||||
local_host_docker_blocked=local_host_docker_blocked,
|
||||
),
|
||||
"session_id": session_id,
|
||||
}
|
||||
|
||||
|
||||
@@ -802,12 +802,17 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
to_delete = []
|
||||
now = datetime.now(timezone.utc)
|
||||
for doc in docs:
|
||||
content = (doc.current_content or "").strip()
|
||||
title_raw = (doc.title or "").strip()
|
||||
title = title_raw.lower()
|
||||
created = doc.created_at
|
||||
if created and created.tzinfo is None:
|
||||
created = created.replace(tzinfo=timezone.utc)
|
||||
|
||||
# Skip freshly created documents to avoid deleting them while the user is actively editing
|
||||
if created and (now - created).total_seconds() < 900: # 15 minutes
|
||||
continue
|
||||
|
||||
content = (doc.current_content or "").strip()
|
||||
title_raw = (doc.title or "").strip()
|
||||
title = title_raw.lower()
|
||||
is_fresh_empty = (
|
||||
not content
|
||||
and created is not None
|
||||
@@ -849,10 +854,6 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
|
||||
to_delete.append(doc); deleted += 1; continue
|
||||
if title in _JUNK_TITLES:
|
||||
to_delete.append(doc); deleted += 1; continue
|
||||
if real_len < 30:
|
||||
to_delete.append(doc); deleted += 1; continue
|
||||
if "\n" not in content and real_len < 50:
|
||||
to_delete.append(doc); deleted += 1; continue
|
||||
|
||||
# Fix empty or placeholder titles on survivors
|
||||
if not title_raw or title_raw == "Untitled":
|
||||
|
||||
@@ -77,6 +77,39 @@ def _normalize_image_endpoint_base(url: str) -> str:
|
||||
return base
|
||||
|
||||
|
||||
def _is_openai_api_base(url: str) -> bool:
|
||||
"""Return True only when url's hostname is exactly api.openai.com."""
|
||||
from urllib.parse import urlsplit
|
||||
try:
|
||||
candidate = url if "://" in url else f"https://{url}"
|
||||
return urlsplit(candidate).hostname == "api.openai.com"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
_GALLERY_ENDPOINT_PATHS = frozenset({
|
||||
"/images/edits",
|
||||
"/images/generations",
|
||||
"/images/harmonize",
|
||||
"/images/img2img",
|
||||
"/images/inpaint",
|
||||
"/images/upscale",
|
||||
"/images/variations",
|
||||
"/sdapi/v1/img2img",
|
||||
})
|
||||
|
||||
|
||||
def _join_checked_gallery_endpoint(base: str, path: str) -> str:
|
||||
"""Append a known-constant gallery path suffix to a validated base URL.
|
||||
|
||||
Rejects paths not in the pre-approved list so arbitrary strings can never
|
||||
be spliced into the URL passed to httpx.
|
||||
"""
|
||||
if path not in _GALLERY_ENDPOINT_PATHS:
|
||||
raise ValueError(f"Unexpected gallery path: {path!r}")
|
||||
return base + path
|
||||
|
||||
|
||||
def _visible_image_endpoint_query(db, owner: str | None):
|
||||
from src.auth_helpers import owner_filter
|
||||
q = db.query(ModelEndpoint).filter(
|
||||
@@ -255,9 +288,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
pass
|
||||
try:
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(500, f"DB commit failed: {e}")
|
||||
logger.exception("gallery_replace: DB commit failed")
|
||||
raise HTTPException(500, "Image update failed")
|
||||
return {"ok": True, "width": img.width, "height": img.height}
|
||||
finally:
|
||||
db.close()
|
||||
@@ -385,8 +419,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
return {"image": data.get("data", [{}])[0].get("b64_json", "")}
|
||||
# Fallback: no upscale endpoint — return error
|
||||
return {"error": f"Upscale endpoint not available ({resp.status_code})"}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
except Exception:
|
||||
logger.exception("ai_upscale: request failed")
|
||||
return {"error": "Upscale request failed"}
|
||||
|
||||
# ---- POST /api/gallery/style-transfer ----
|
||||
@router.post("/api/gallery/style-transfer")
|
||||
@@ -431,8 +466,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
if img_data:
|
||||
return {"image": img_data}
|
||||
return {"error": f"Style transfer failed ({resp.status_code})"}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
except Exception:
|
||||
logger.exception("style_transfer: request failed")
|
||||
return {"error": "Style transfer failed"}
|
||||
|
||||
# ---- GET /api/gallery/tags ----
|
||||
@router.get("/api/gallery/tags")
|
||||
@@ -588,9 +624,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
"tags": sorted(all_tags),
|
||||
"models": all_models,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch gallery library: {e}")
|
||||
raise HTTPException(500, f"Failed to fetch gallery library: {e}")
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch gallery library")
|
||||
raise HTTPException(500, "Failed to fetch gallery library")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -766,9 +802,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
return _image_to_dict(img)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(500, str(e))
|
||||
logger.exception("patch_gallery_image: update failed")
|
||||
raise HTTPException(500, "Image update failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -845,9 +882,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
cleared += 1
|
||||
db.commit()
|
||||
return {"ok": True, "cleared": cleared}
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(500, str(e))
|
||||
logger.exception("clear_gallery_user_tags: failed")
|
||||
raise HTTPException(500, "Tag update failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -871,9 +909,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
cleared += 1
|
||||
db.commit()
|
||||
return {"ok": True, "cleared": cleared}
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(500, str(e))
|
||||
logger.exception("clear_gallery_ai_tags: failed")
|
||||
raise HTTPException(500, "Tag update failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -909,9 +948,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
img.tags = ', '.join(cleaned)
|
||||
db.commit()
|
||||
return {"ok": True, "rows_touched": rows_touched, "tags_removed": tags_removed}
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(500, str(e))
|
||||
logger.exception("dedupe_gallery_tags: failed")
|
||||
raise HTTPException(500, "Tag deduplication failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -1029,9 +1069,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
return {"status": "deleted", "id": image_id}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise HTTPException(500, str(e))
|
||||
logger.exception("delete_gallery_image: failed")
|
||||
raise HTTPException(500, "Image deletion failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -1044,21 +1085,22 @@ def setup_gallery_routes() -> APIRouter:
|
||||
import httpx
|
||||
user = require_privilege(request, "can_generate_images")
|
||||
body = await request.json()
|
||||
# Use endpoint from request body (editor dropdown) or fall back to DB lookup
|
||||
base = (body.pop("_endpoint", "") or "").rstrip("/")
|
||||
# Use endpoint from request body (editor dropdown) or fall back to DB lookup.
|
||||
# Store as requested_base to avoid carrying user input into the outbound request.
|
||||
requested_base = (body.pop("_endpoint", "") or "").rstrip("/")
|
||||
# SSRF hardening: validate a client-supplied endpoint before any
|
||||
# outbound request (mirrors routes/embedding_routes.py).
|
||||
if base:
|
||||
if requested_base:
|
||||
from src.url_safety import check_outbound_url
|
||||
ok, reason = check_outbound_url(
|
||||
base,
|
||||
requested_base,
|
||||
block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true",
|
||||
)
|
||||
if not ok:
|
||||
raise HTTPException(400, f"Rejected endpoint URL: {reason}")
|
||||
chosen_model = (body.pop("_model", "") or "").strip()
|
||||
api_key = None
|
||||
if not base:
|
||||
if not requested_base:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _first_visible_image_endpoint(db, user)
|
||||
@@ -1069,32 +1111,23 @@ def setup_gallery_routes() -> APIRouter:
|
||||
finally:
|
||||
db.close()
|
||||
else:
|
||||
# Pull api_key from the matching DB row so OpenAI auth works.
|
||||
# Users may have stored base_url with/without /v1 suffix and with/without
|
||||
# trailing slash, so compare normalized forms.
|
||||
def _norm_url(u: str) -> str:
|
||||
if not u:
|
||||
return u
|
||||
u = u.rstrip("/")
|
||||
if u.endswith("/v1"):
|
||||
u = u[:-3]
|
||||
return u
|
||||
_target = _norm_url(base)
|
||||
# Resolve the client-supplied base to a registered visible endpoint.
|
||||
# Admins are not exempted — gallery proxy routes must use a DB row
|
||||
# so the outbound URL never depends directly on request-body input.
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _visible_image_endpoint_for_base(db, _target, user)
|
||||
if ep:
|
||||
base = (ep.base_url or base).rstrip("/")
|
||||
api_key = ep.api_key
|
||||
elif user and not _current_user_is_admin(request, user):
|
||||
ep = _visible_image_endpoint_for_base(db, requested_base, user)
|
||||
if not ep:
|
||||
raise HTTPException(403, "Choose a registered image endpoint")
|
||||
base = ep.base_url.rstrip("/")
|
||||
api_key = ep.api_key
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if not base.endswith("/v1"):
|
||||
base += "/v1"
|
||||
|
||||
is_openai = "api.openai.com" in base
|
||||
is_openai = _is_openai_api_base(base)
|
||||
|
||||
if is_openai:
|
||||
# OpenAI path: /v1/images/edits with gpt-image-1.
|
||||
@@ -1131,8 +1164,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
mask_buf.seek(0)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Failed to prepare OpenAI request: {e}")
|
||||
except Exception:
|
||||
logger.exception("inpaint_proxy: failed to prepare OpenAI request")
|
||||
raise HTTPException(400, "Failed to prepare inpaint request")
|
||||
|
||||
width = int(body.get("width") or 1024)
|
||||
height = int(body.get("height") or 1024)
|
||||
@@ -1163,9 +1197,10 @@ def setup_gallery_routes() -> APIRouter:
|
||||
headers = {"Authorization": f"Bearer {api_key}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
r = await client.post(f"{base}/images/edits", headers=headers, data=data, files=files)
|
||||
r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), headers=headers, data=data, files=files)
|
||||
if r.status_code != 200:
|
||||
raise HTTPException(r.status_code, f"OpenAI edit failed: {r.text[:300]}")
|
||||
logger.error("inpaint_proxy OpenAI edit: status %s", r.status_code)
|
||||
raise HTTPException(r.status_code, "OpenAI edit failed")
|
||||
result = r.json()
|
||||
raw_b64 = None
|
||||
if result.get("data"):
|
||||
@@ -1212,16 +1247,18 @@ def setup_gallery_routes() -> APIRouter:
|
||||
if chosen_model:
|
||||
body["model"] = chosen_model
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
r = await client.post(f"{base}/images/inpaint", json=body)
|
||||
r = await client.post(_join_checked_gallery_endpoint(base, "/images/inpaint"), json=body)
|
||||
if r.status_code != 200:
|
||||
raise HTTPException(r.status_code, f"Inpaint failed: {r.text[:200]}")
|
||||
logger.error("inpaint_proxy diffusion: status %s", r.status_code)
|
||||
raise HTTPException(r.status_code, "Inpaint request failed")
|
||||
return r.json()
|
||||
except httpx.TimeoutException:
|
||||
raise HTTPException(504, "Inpaint request timed out (120s)")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"Inpaint error: {str(e)}")
|
||||
except Exception:
|
||||
logger.exception("inpaint_proxy: request failed")
|
||||
raise HTTPException(502, "Inpaint request failed")
|
||||
|
||||
# ---- POST /api/image/harmonize — proper img2img call ----
|
||||
# Earlier version routed through inpaint with a full-white mask, but
|
||||
@@ -1243,24 +1280,23 @@ def setup_gallery_routes() -> APIRouter:
|
||||
if not image_b64:
|
||||
raise HTTPException(400, "No image provided")
|
||||
|
||||
endpoint = (body.get("_endpoint") or "").rstrip("/")
|
||||
requested_base = (body.get("_endpoint") or "").rstrip("/")
|
||||
# SSRF hardening: a client-supplied endpoint is fetched server-side
|
||||
# below, so validate it first (mirrors routes/embedding_routes.py).
|
||||
# Local-first means loopback/LAN is allowed by default; the cloud
|
||||
# metadata range and non-HTTP(S) schemes are always rejected.
|
||||
if endpoint:
|
||||
if requested_base:
|
||||
from src.url_safety import check_outbound_url
|
||||
ok, reason = check_outbound_url(
|
||||
endpoint,
|
||||
requested_base,
|
||||
block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true",
|
||||
)
|
||||
if not ok:
|
||||
raise HTTPException(400, f"Rejected endpoint URL: {reason}")
|
||||
model = (body.get("_model") or "").strip()
|
||||
|
||||
base = endpoint
|
||||
api_key = None
|
||||
if not base:
|
||||
if not requested_base:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _first_visible_image_endpoint(db, user)
|
||||
@@ -1271,14 +1307,16 @@ def setup_gallery_routes() -> APIRouter:
|
||||
finally:
|
||||
db.close()
|
||||
else:
|
||||
# Resolve the client-supplied base to a registered visible endpoint.
|
||||
# Admins are not exempted — gallery proxy routes must use a DB row
|
||||
# so the outbound URL never depends directly on request-body input.
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ep = _visible_image_endpoint_for_base(db, base, user)
|
||||
if ep:
|
||||
base = (ep.base_url or base).rstrip("/")
|
||||
api_key = ep.api_key
|
||||
elif user and not _current_user_is_admin(request, user):
|
||||
ep = _visible_image_endpoint_for_base(db, requested_base, user)
|
||||
if not ep:
|
||||
raise HTTPException(403, "Choose a registered image endpoint")
|
||||
base = ep.base_url.rstrip("/")
|
||||
api_key = ep.api_key
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -1313,7 +1351,7 @@ def setup_gallery_routes() -> APIRouter:
|
||||
# source. Earlier hack (alpha-blend the regen back at `strength`)
|
||||
# produced visibly broken results, so we refuse and tell the
|
||||
# user to spin up a real diffusion endpoint instead.
|
||||
if "api.openai.com" in base:
|
||||
if _is_openai_api_base(base):
|
||||
raise HTTPException(400,
|
||||
"Harmonize needs a diffusion server that supports img2img "
|
||||
"(SD WebUI / Forge / Comfy). OpenAI's API doesn't expose "
|
||||
@@ -1378,14 +1416,16 @@ def setup_gallery_routes() -> APIRouter:
|
||||
# 1024×1024 inference pass on slower setups.
|
||||
async with httpx.AsyncClient(timeout=240) as client:
|
||||
for path, kind, payload in candidates:
|
||||
target = base_root + path if path.startswith("/sdapi") else base + path
|
||||
_effective_base = base_root if path.startswith("/sdapi") else base
|
||||
target = _join_checked_gallery_endpoint(_effective_base, path)
|
||||
try:
|
||||
r = await client.post(target, json=payload, headers=headers)
|
||||
if r.status_code == 404:
|
||||
last_err = f"{path}: 404"
|
||||
continue # try next variant
|
||||
if r.status_code != 200:
|
||||
last_err = f"{path}: {r.status_code} {r.text[:120]}"
|
||||
logger.warning("harmonize: %s returned %s", path, r.status_code)
|
||||
last_err = f"{path}: {r.status_code}"
|
||||
continue
|
||||
data = r.json()
|
||||
# Normalise return shape.
|
||||
@@ -1394,8 +1434,8 @@ def setup_gallery_routes() -> APIRouter:
|
||||
# surface it now instead of trying the other routes
|
||||
# (otherwise the real error gets buried under 404s).
|
||||
if data.get("error") and not data.get("image"):
|
||||
raise HTTPException(502,
|
||||
f"Diffusion server error at {path}: {data['error']}")
|
||||
logger.warning("harmonize: server error at %s: %s", path, data.get("error"))
|
||||
raise HTTPException(502, f"Diffusion server error at {path}")
|
||||
if data.get("image"):
|
||||
return {"image": data["image"]}
|
||||
if data.get("images") and isinstance(data["images"], list):
|
||||
@@ -1415,15 +1455,15 @@ def setup_gallery_routes() -> APIRouter:
|
||||
if img_b64:
|
||||
return {"image": img_b64}
|
||||
last_err = f"{path}: server returned no image"
|
||||
except httpx.ConnectError as e:
|
||||
raise HTTPException(502, f"Can't reach diffusion server at {base}: {e}")
|
||||
except httpx.ConnectError:
|
||||
logger.warning("harmonize: can't reach diffusion server at %s", base)
|
||||
raise HTTPException(502, "Can't reach diffusion server")
|
||||
except httpx.TimeoutException:
|
||||
raise HTTPException(504, "Harmonize timed out (240s) — restart the diffusion server or lower Color match / disable Seam fix")
|
||||
raise HTTPException(502,
|
||||
f"None of the img2img routes worked on {base}. "
|
||||
f"Last response: {last_err or 'unknown'}. "
|
||||
"Your diffusion server needs to expose one of /v1/images/harmonize, "
|
||||
"/v1/images/img2img, /v1/images/variations, or /sdapi/v1/img2img.")
|
||||
"No supported img2img route responded. "
|
||||
"Your diffusion server needs to expose one of: "
|
||||
"/v1/images/harmonize, /v1/images/img2img, /v1/images/variations, /sdapi/v1/img2img.")
|
||||
|
||||
# ---- POST /api/image/sharpen ----
|
||||
@router.post("/api/image/sharpen")
|
||||
@@ -1467,8 +1507,8 @@ def setup_gallery_routes() -> APIRouter:
|
||||
import base64, io
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
except ImportError as e:
|
||||
raise HTTPException(500, f"Server missing dependency: {e}")
|
||||
except ImportError:
|
||||
raise HTTPException(500, "Server missing a required dependency")
|
||||
# Decode source image (RGB; Real-ESRGAN doesn't preserve alpha).
|
||||
img_bytes = base64.b64decode(image_b64)
|
||||
src = Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
||||
@@ -1495,9 +1535,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
buf = io.BytesIO()
|
||||
out_img.save(buf, format="PNG")
|
||||
return {"image": base64.b64encode(buf.getvalue()).decode()}
|
||||
except Exception as e:
|
||||
logger.warning(f"Denoise failed: {e}")
|
||||
return {"error": f"Denoise failed: {e}"}
|
||||
except Exception:
|
||||
logger.warning("Denoise failed", exc_info=True)
|
||||
return {"error": "Denoise failed"}
|
||||
|
||||
# ---- POST /api/image/upscale-local ----
|
||||
# Local Real-ESRGAN upscale (2× or 4×). Self-contained — no diffusion
|
||||
@@ -1518,8 +1558,8 @@ def setup_gallery_routes() -> APIRouter:
|
||||
import base64, io
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
except ImportError as e:
|
||||
raise HTTPException(500, f"Server missing dependency: {e}")
|
||||
except ImportError:
|
||||
raise HTTPException(500, "Server missing a required dependency")
|
||||
img_bytes = base64.b64decode(image_b64)
|
||||
src = Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
||||
try:
|
||||
@@ -1543,9 +1583,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
buf = io.BytesIO()
|
||||
out_img.save(buf, format="PNG")
|
||||
return {"image": base64.b64encode(buf.getvalue()).decode()}
|
||||
except Exception as e:
|
||||
logger.warning(f"Upscale failed: {e}")
|
||||
return {"error": f"Upscale failed: {e}"}
|
||||
except Exception:
|
||||
logger.warning("AI upscale failed", exc_info=True)
|
||||
return {"error": "AI upscale failed"}
|
||||
|
||||
# ---- POST /api/image/remove-bg ----
|
||||
@router.post("/api/image/remove-bg")
|
||||
@@ -1703,8 +1743,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
buf = io.BytesIO()
|
||||
enhanced.save(buf, format="PNG")
|
||||
return {"image": base64.b64encode(buf.getvalue()).decode(), "method": "pil"}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Face enhancement failed: {str(e)}")
|
||||
except Exception:
|
||||
logger.exception("enhance_face: failed")
|
||||
raise HTTPException(500, "Face enhancement failed")
|
||||
|
||||
# ---- Album management (path-param routes) ----
|
||||
|
||||
@@ -1899,9 +1940,8 @@ def setup_gallery_routes() -> APIRouter:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
resp = await client.post(chat_url, json=payload, headers=h)
|
||||
if resp.status_code != 200:
|
||||
body = resp.text[:500]
|
||||
logger.error(f"Vision model {resp.status_code}: {body}")
|
||||
return {"error": f"Vision model returned {resp.status_code}: {body[:200]}"}
|
||||
logger.error("ai_tag vision model: status %s: %s", resp.status_code, resp.text[:500])
|
||||
return {"error": "Vision model request failed"}
|
||||
data = resp.json()
|
||||
# Anthropic returns content[0].text, OpenAI returns choices[0].message.content
|
||||
if provider == "anthropic":
|
||||
@@ -1917,9 +1957,9 @@ def setup_gallery_routes() -> APIRouter:
|
||||
return {"ok": True, "ai_tags": tag_str}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"AI tagging failed: {e}")
|
||||
return {"error": str(e)}
|
||||
except Exception:
|
||||
logger.exception("AI tagging failed")
|
||||
return {"error": "Auto-tagging failed"}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Memory route domain package (slice 2c, #4082/#4071).
|
||||
|
||||
Contains memory_routes.py, migrated from the flat routes/ directory.
|
||||
Backward-compat shim at routes/memory_routes.py re-exports from here.
|
||||
"""
|
||||
@@ -0,0 +1,552 @@
|
||||
# routes/memory_routes.py
|
||||
from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
|
||||
from typing import Dict, Any, Optional, List
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
|
||||
# Strips one prefix per call so import-from-LLM-output doesn't leave the
|
||||
# numbering inside the saved memory text. Bullet markers (-, *, •) are
|
||||
# also peeled here for the same reason.
|
||||
_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
|
||||
|
||||
|
||||
def _strip_list_prefix(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
|
||||
|
||||
from services.memory import MemoryManager
|
||||
from core.session_manager import SessionManager
|
||||
from src.request_models import MemoryAddRequest
|
||||
from core.database import SessionLocal
|
||||
from src.llm_core import llm_call_async
|
||||
from services.memory.memory_extractor import audit_memories
|
||||
from src.auth_helpers import get_current_user, require_user
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
from src.task_endpoint import resolve_task_endpoint
|
||||
from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
|
||||
"""Set up memory-related routes."""
|
||||
router = APIRouter(prefix="/api/memory", tags=["memory"])
|
||||
|
||||
def _owner(request: Request) -> Optional[str]:
|
||||
return get_current_user(request)
|
||||
|
||||
def _assert_session_owner(session_obj, user):
|
||||
"""SECURITY: 404 if the caller does not own this session.
|
||||
|
||||
SessionManager.get_session is NOT owner-scoped — it returns any
|
||||
session by id. These routes accept a caller-supplied session id, so
|
||||
without this gate a user could target another tenant's session and
|
||||
leak their chat history, their session-scoped LLM credentials, or the
|
||||
session title. Mirrors session_routes / webhook_routes ownership.
|
||||
"""
|
||||
if user is not None and getattr(session_obj, "owner", None) != user:
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
def _verify_memory_owner(memory: dict, user: Optional[str]):
|
||||
"""Raise 404 if user doesn't own this memory.
|
||||
|
||||
SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
|
||||
allowed any user to read/edit/delete memories with an empty/null owner
|
||||
field, which leaked legacy data across the multi-user deploy.
|
||||
"""
|
||||
if user is None:
|
||||
return # Auth disabled
|
||||
if memory.get("owner") != user:
|
||||
raise HTTPException(404, "Memory not found")
|
||||
|
||||
@router.post("/debug")
|
||||
def debug_memory_relevance(request: Request, query: str = Form(...)):
|
||||
"""Debug which memories would be triggered for a query"""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
|
||||
|
||||
return {
|
||||
"query": query,
|
||||
"total_memories": len(memories),
|
||||
"relevant_count": len(relevant),
|
||||
"relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
|
||||
for m in relevant]
|
||||
}
|
||||
|
||||
@router.post("/add", response_model=Dict[str, Any])
|
||||
async def api_add_memory(
|
||||
request: Request,
|
||||
memory_data: Optional[MemoryAddRequest] = None
|
||||
):
|
||||
"""Add a new memory entry with optional category, source, and session reference."""
|
||||
from src.auth_helpers import require_privilege
|
||||
require_privilege(request, "can_manage_memory")
|
||||
if memory_data is None:
|
||||
form = await request.form()
|
||||
memory_data = MemoryAddRequest(
|
||||
text=form.get("text"),
|
||||
category=form.get("category", "fact"),
|
||||
source=form.get("source", "user"),
|
||||
session_id=form.get("session_id")
|
||||
)
|
||||
|
||||
user = _owner(request)
|
||||
text = (memory_data.text or "").strip()
|
||||
if not text:
|
||||
raise HTTPException(400, "empty memory")
|
||||
user_mem = memory_manager.load(owner=user)
|
||||
if memory_manager.find_duplicates(text, user_mem):
|
||||
return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
|
||||
|
||||
if memory_data.session_id:
|
||||
try:
|
||||
session_obj = session_manager.get_session(memory_data.session_id)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "Session not found")
|
||||
_assert_session_owner(session_obj, user)
|
||||
|
||||
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
|
||||
if memory_data.session_id:
|
||||
new_entry["session_id"] = memory_data.session_id
|
||||
all_mem = memory_manager.load_all()
|
||||
all_mem.append(new_entry)
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index
|
||||
if memory_vector and memory_vector.healthy:
|
||||
memory_vector.add(new_entry["id"], text)
|
||||
try:
|
||||
from src.event_bus import fire_event
|
||||
fire_event("memory_added", user)
|
||||
except Exception:
|
||||
logger.debug("memory_added event dispatch failed", exc_info=True)
|
||||
return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
|
||||
|
||||
@router.get("")
|
||||
def api_get_memory(request: Request):
|
||||
"""Return all memory entries with their metadata."""
|
||||
user = _owner(request)
|
||||
return {"memory": memory_manager.load(owner=user)}
|
||||
|
||||
@router.post("/search")
|
||||
def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
|
||||
"""Search across all memories with optional filters."""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
|
||||
if session_id:
|
||||
memories = [m for m in memories if m.get("session_id") == session_id]
|
||||
|
||||
if category:
|
||||
memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
|
||||
|
||||
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
|
||||
|
||||
return {"memories": relevant, "total": len(relevant), "query": query}
|
||||
|
||||
@router.get("/timeline")
|
||||
def memory_timeline(request: Request):
|
||||
"""Get memories in chronological order with source session information."""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
|
||||
|
||||
results = []
|
||||
for memory in sorted_memories:
|
||||
if "timestamp" in memory:
|
||||
try:
|
||||
dt = datetime.fromtimestamp(memory["timestamp"])
|
||||
memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
except (ValueError, OSError, OverflowError):
|
||||
memory["timestamp_str"] = "Unknown"
|
||||
else:
|
||||
memory["timestamp_str"] = "Unknown"
|
||||
|
||||
session_id = memory.get("session_id")
|
||||
if session_id and session_id in session_manager.sessions:
|
||||
try:
|
||||
session = session_manager.get_session(session_id)
|
||||
if session:
|
||||
_assert_session_owner(session, user)
|
||||
memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
|
||||
except KeyError:
|
||||
memory["session_name"] = "Unknown"
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
memory["session_name"] = "Unknown"
|
||||
else:
|
||||
memory["session_name"] = "Unknown"
|
||||
|
||||
results.append(memory)
|
||||
|
||||
return {"timeline": results, "total": len(results)}
|
||||
|
||||
@router.get("/by-session/{session_id}")
|
||||
def get_memory_by_session(request: Request, session_id: str):
|
||||
"""Get all memories associated with a specific session."""
|
||||
user = _owner(request)
|
||||
try:
|
||||
_session_obj = session_manager.get_session(session_id)
|
||||
except KeyError:
|
||||
raise HTTPException(404, f"Session {session_id} not found")
|
||||
_assert_session_owner(_session_obj, user)
|
||||
memories = memory_manager.load(owner=user)
|
||||
session_memories = [m for m in memories if m.get("session_id") == session_id]
|
||||
|
||||
session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
|
||||
|
||||
try:
|
||||
session = session_manager.get_session(session_id)
|
||||
session_name = session.name if session else f"Session {session_id[:6]}"
|
||||
except KeyError:
|
||||
session_name = f"Session {session_id[:6]}"
|
||||
|
||||
for memory in session_memories:
|
||||
memory["session_name"] = session_name
|
||||
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"session_name": session_name,
|
||||
"memory_count": len(session_memories),
|
||||
"memories": session_memories
|
||||
}
|
||||
|
||||
@router.post("/extract")
|
||||
async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
|
||||
"""Analyze a session's chat history and return memory suggestions."""
|
||||
require_user(request)
|
||||
try:
|
||||
sess = session_manager.get_session(session)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "Session not found")
|
||||
_assert_session_owner(sess, _owner(request))
|
||||
|
||||
system_msg = {
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a helpful assistant. Analyze the entire conversation history provided and extract any "
|
||||
"useful factual statements, contacts, addresses, phone numbers, or other information that the user "
|
||||
"might want to remember for future interactions. Return each piece of information as a JSON object "
|
||||
"with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
|
||||
"Only include information that is specific and likely to be useful later."
|
||||
),
|
||||
}
|
||||
messages = [system_msg] + sess.get_context_messages()
|
||||
|
||||
t_url, t_model, t_headers = resolve_task_endpoint(
|
||||
sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
|
||||
)
|
||||
|
||||
try:
|
||||
suggestion_text = await llm_call_async(
|
||||
t_url,
|
||||
t_model,
|
||||
messages,
|
||||
temperature=0.2,
|
||||
max_tokens=500,
|
||||
headers=t_headers,
|
||||
)
|
||||
try:
|
||||
suggestions = json.loads(suggestion_text)
|
||||
if isinstance(suggestions, list):
|
||||
suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
|
||||
else:
|
||||
suggestions = []
|
||||
except json.JSONDecodeError:
|
||||
suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
|
||||
|
||||
return {"suggestions": [s for s in suggestions if s]}
|
||||
except Exception as e:
|
||||
logger.error(f"LLM memory extraction failed (session {session}): {e}")
|
||||
fallback = memory_manager.extract_memory_from_chat(sess.history, session)
|
||||
return {"suggestions": [item["text"] for item in fallback]}
|
||||
|
||||
@router.post("/audit")
|
||||
async def api_audit_memories(request: Request, session: str = Form(None)):
|
||||
"""Deduplicate and consolidate memories via LLM.
|
||||
|
||||
Uses task/utility/default settings through the shared resolver, with
|
||||
the active session as fallback when no task or utility model is set.
|
||||
Returns before and after memory counts.
|
||||
"""
|
||||
user = _owner(request)
|
||||
fallback_url = fallback_model = None
|
||||
fallback_headers = None
|
||||
if session:
|
||||
try:
|
||||
sess = session_manager.get_session(session)
|
||||
_assert_session_owner(sess, user)
|
||||
fallback_url = sess.endpoint_url
|
||||
fallback_model = sess.model
|
||||
fallback_headers = sess.headers
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
endpoint_url, model, headers = resolve_task_endpoint(
|
||||
fallback_url, fallback_model, fallback_headers, owner=user
|
||||
)
|
||||
|
||||
if not endpoint_url or not model:
|
||||
raise HTTPException(400, "No default model configured — set one in Settings")
|
||||
|
||||
result = await audit_memories(
|
||||
memory_manager,
|
||||
memory_vector,
|
||||
endpoint_url,
|
||||
model,
|
||||
headers,
|
||||
owner=user,
|
||||
)
|
||||
|
||||
if "error" in result and "before" not in result:
|
||||
raise HTTPException(502, f"Audit failed: {result['error']}")
|
||||
|
||||
return {
|
||||
"ok": "error" not in result,
|
||||
"before": result.get("before", 0),
|
||||
"after": result.get("after", 0),
|
||||
"removed": result.get("before", 0) - result.get("after", 0),
|
||||
# True when the audit skipped the LLM because nothing changed
|
||||
# since the last tidy. Frontend already says "Already clean"
|
||||
# for removed==0, so this is here for future use / debugging.
|
||||
"already_tidy": bool(result.get("already_tidy")),
|
||||
}
|
||||
|
||||
@router.post("/import")
|
||||
async def import_memories_from_file(
|
||||
request: Request,
|
||||
session: str | None = Form(None),
|
||||
file: UploadFile = File(...)
|
||||
):
|
||||
"""Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
|
||||
from src.auth_helpers import require_privilege
|
||||
require_privilege(request, "can_manage_memory")
|
||||
|
||||
endpoint_url = None
|
||||
model = None
|
||||
headers = {}
|
||||
|
||||
user = _owner(request)
|
||||
|
||||
if session:
|
||||
try:
|
||||
sess = session_manager.get_session(session)
|
||||
_assert_session_owner(sess, user)
|
||||
except KeyError:
|
||||
sess = None
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
sess = None
|
||||
|
||||
if sess is None:
|
||||
logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
|
||||
endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
|
||||
else:
|
||||
endpoint_url, model, headers = resolve_task_endpoint(
|
||||
sess.endpoint_url, sess.model, sess.headers, owner=user
|
||||
)
|
||||
else:
|
||||
endpoint_url, model, headers = resolve_task_endpoint(owner=user)
|
||||
|
||||
if not endpoint_url or not model:
|
||||
raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
|
||||
|
||||
content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
|
||||
filename = file.filename or "upload"
|
||||
_, ext = os.path.splitext(filename.lower())
|
||||
|
||||
allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
|
||||
if ext not in allowed:
|
||||
raise HTTPException(400, f"Unsupported file type: {ext}")
|
||||
|
||||
# Extract text based on file type
|
||||
if ext == ".pdf":
|
||||
from src.document_processor import _process_pdf
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
text = _process_pdf(tmp_path, owner=_owner(request))
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
else:
|
||||
try:
|
||||
text = content.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
from charset_normalizer import detect
|
||||
encoding = (detect(content) or {}).get("encoding") or "utf-8"
|
||||
text = content.decode(encoding, errors="replace")
|
||||
|
||||
if not text.strip():
|
||||
return {"suggestions": [], "message": "No readable content found"}
|
||||
|
||||
# Fast path: a .json upload that already looks like a memories export
|
||||
# (list of {text, category, ...} dicts, or list of strings) round-trips
|
||||
# directly without spending an LLM call to re-extract its own output.
|
||||
# Without this, re-importing a memories.json from another account
|
||||
# ran the file through the extractor, which often re-emitted the
|
||||
# entries as a numbered list (and the numbering leaked into the
|
||||
# `text` field).
|
||||
if ext == ".json":
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
if isinstance(parsed, list) and parsed:
|
||||
direct = []
|
||||
for item in parsed:
|
||||
if isinstance(item, dict) and item.get("text"):
|
||||
direct.append({
|
||||
"text": _strip_list_prefix(str(item["text"])),
|
||||
"category": item.get("category") or "fact",
|
||||
})
|
||||
elif isinstance(item, str) and item.strip():
|
||||
direct.append({
|
||||
"text": _strip_list_prefix(item.strip()),
|
||||
"category": "fact",
|
||||
})
|
||||
if direct:
|
||||
return {"suggestions": direct, "filename": filename}
|
||||
|
||||
# Truncate very long documents
|
||||
if len(text) > 15000:
|
||||
text = text[:15000] + "\n[Truncated]"
|
||||
|
||||
# Send to LLM for memory extraction
|
||||
import_prompt = (
|
||||
"You are a memory extraction assistant. The user uploaded a document. "
|
||||
"Analyze the text below and extract specific, useful facts — things like "
|
||||
"names, preferences, jobs, locations, relationships, opinions, projects, "
|
||||
"goals, contacts, or any other personal details worth remembering.\n\n"
|
||||
"Rules:\n"
|
||||
"- Each fact should be a short, self-contained statement\n"
|
||||
"- Do NOT extract generic knowledge\n"
|
||||
"- Focus on personal, memorable information\n"
|
||||
"- If there are no useful facts, return an empty array\n\n"
|
||||
"Return a JSON array of objects with 'text' and 'category' fields.\n"
|
||||
"Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
|
||||
"Return ONLY valid JSON, no markdown fences."
|
||||
)
|
||||
|
||||
try:
|
||||
raw = await llm_call_async(
|
||||
endpoint_url,
|
||||
model,
|
||||
[
|
||||
{"role": "system", "content": import_prompt},
|
||||
{"role": "user", "content": f"Document: {filename}\n\n{text}"},
|
||||
],
|
||||
temperature=0.2,
|
||||
max_tokens=2000,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Parse JSON
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
|
||||
|
||||
suggestions = json.loads(raw)
|
||||
if isinstance(suggestions, list):
|
||||
normalized = []
|
||||
for s in suggestions:
|
||||
if not s:
|
||||
continue
|
||||
if isinstance(s, dict):
|
||||
s = dict(s)
|
||||
if s.get("text"):
|
||||
s["text"] = _strip_list_prefix(str(s["text"]))
|
||||
normalized.append(s)
|
||||
else:
|
||||
normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
|
||||
suggestions = normalized
|
||||
else:
|
||||
suggestions = []
|
||||
|
||||
return {"suggestions": suggestions, "filename": filename}
|
||||
|
||||
except json.JSONDecodeError:
|
||||
# Fallback: split by lines, stripping any "1.", "2)" markdown-list
|
||||
# numbering the model added so saved memories don't keep the prefix.
|
||||
lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
|
||||
return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
|
||||
except Exception as e:
|
||||
logger.error(f"Memory import extraction failed: {e}")
|
||||
raise HTTPException(502, f"LLM extraction failed: {str(e)}")
|
||||
|
||||
@router.post("/{memory_id}/pin")
|
||||
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
|
||||
"""Pin or unpin a memory. Pinned memories are always included in context."""
|
||||
user = _owner(request)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
all_mem[i]["pinned"] = pinned
|
||||
memory_manager.save(all_mem)
|
||||
return {"ok": True, "pinned": pinned}
|
||||
raise HTTPException(404, f"Memory item {memory_id} not found")
|
||||
|
||||
# Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
|
||||
@router.get("/{memory_id}")
|
||||
def get_memory_item(request: Request, memory_id: str):
|
||||
"""Get a specific memory item by ID."""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
for memory in memories:
|
||||
if memory["id"] == memory_id:
|
||||
return {"memory": memory}
|
||||
|
||||
raise HTTPException(404, "Memory not found")
|
||||
|
||||
@router.put("/{memory_id}")
|
||||
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
|
||||
"""Update an existing memory item with new text and optional category."""
|
||||
user = _owner(request)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
all_mem[i]["text"] = text.strip()
|
||||
if category:
|
||||
all_mem[i]["category"] = category
|
||||
all_mem[i]["timestamp"] = int(time.time())
|
||||
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index (remove old, add updated)
|
||||
if memory_vector and memory_vector.healthy:
|
||||
memory_vector.remove(memory_id)
|
||||
memory_vector.add(memory_id, text.strip())
|
||||
return {"ok": True, "message": "Memory updated successfully"}
|
||||
|
||||
raise HTTPException(404, f"Memory item {memory_id} not found")
|
||||
|
||||
@router.delete("/{memory_id}")
|
||||
def delete_memory(request: Request, memory_id: str):
|
||||
"""Delete a memory item by its ID."""
|
||||
user = _owner(request)
|
||||
all_mem = memory_manager.load_all()
|
||||
|
||||
# Find and verify ownership before deleting
|
||||
target = next((m for m in all_mem if m["id"] == memory_id), None)
|
||||
if not target:
|
||||
raise HTTPException(404, f"Memory item {memory_id} not found")
|
||||
_verify_memory_owner(target, user)
|
||||
|
||||
all_mem = [m for m in all_mem if m["id"] != memory_id]
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index
|
||||
if memory_vector and memory_vector.healthy:
|
||||
memory_vector.remove(memory_id)
|
||||
return {"ok": True, "message": "Memory deleted successfully"}
|
||||
|
||||
return router
|
||||
+14
-548
@@ -1,552 +1,18 @@
|
||||
# routes/memory_routes.py
|
||||
from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
|
||||
from typing import Dict, Any, Optional, List
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
import logging
|
||||
"""Backward-compat shim — canonical location is routes/memory/memory_routes.py.
|
||||
|
||||
# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
|
||||
# Strips one prefix per call so import-from-LLM-output doesn't leave the
|
||||
# numbering inside the saved memory text. Bullet markers (-, *, •) are
|
||||
# also peeled here for the same reason.
|
||||
_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
|
||||
This module is replaced in ``sys.modules`` by the canonical module object so
|
||||
that ``import routes.memory_routes``, ``from routes.memory_routes import X``,
|
||||
``importlib.import_module("routes.memory_routes")``, and
|
||||
``monkeypatch.setattr(routes.memory_routes, "ATTR", ...)`` (used by
|
||||
test_memory_routes_session_owner.py and test_memory_owner_isolation.py via
|
||||
``import ... as mr`` + ``setattr(mr, ...)``) all operate on the *same* object
|
||||
the application actually uses. Keeps existing import paths working after
|
||||
slice 2c (#4082/#4071). Source-introspection tests read the canonical file
|
||||
by path.
|
||||
"""
|
||||
|
||||
import sys as _sys
|
||||
|
||||
def _strip_list_prefix(text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
return _LIST_PREFIX_RE.sub("", text, count=1).strip()
|
||||
from routes.memory import memory_routes as _canonical # noqa: F401
|
||||
|
||||
from services.memory import MemoryManager
|
||||
from core.session_manager import SessionManager
|
||||
from src.request_models import MemoryAddRequest
|
||||
from core.database import SessionLocal
|
||||
from src.llm_core import llm_call_async
|
||||
from services.memory.memory_extractor import audit_memories
|
||||
from src.auth_helpers import get_current_user, require_user
|
||||
from src.endpoint_resolver import resolve_endpoint
|
||||
from src.task_endpoint import resolve_task_endpoint
|
||||
from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
|
||||
"""Set up memory-related routes."""
|
||||
router = APIRouter(prefix="/api/memory", tags=["memory"])
|
||||
|
||||
def _owner(request: Request) -> Optional[str]:
|
||||
return get_current_user(request)
|
||||
|
||||
def _assert_session_owner(session_obj, user):
|
||||
"""SECURITY: 404 if the caller does not own this session.
|
||||
|
||||
SessionManager.get_session is NOT owner-scoped — it returns any
|
||||
session by id. These routes accept a caller-supplied session id, so
|
||||
without this gate a user could target another tenant's session and
|
||||
leak their chat history, their session-scoped LLM credentials, or the
|
||||
session title. Mirrors session_routes / webhook_routes ownership.
|
||||
"""
|
||||
if user is not None and getattr(session_obj, "owner", None) != user:
|
||||
raise HTTPException(404, "Session not found")
|
||||
|
||||
def _verify_memory_owner(memory: dict, user: Optional[str]):
|
||||
"""Raise 404 if user doesn't own this memory.
|
||||
|
||||
SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
|
||||
allowed any user to read/edit/delete memories with an empty/null owner
|
||||
field, which leaked legacy data across the multi-user deploy.
|
||||
"""
|
||||
if user is None:
|
||||
return # Auth disabled
|
||||
if memory.get("owner") != user:
|
||||
raise HTTPException(404, "Memory not found")
|
||||
|
||||
@router.post("/debug")
|
||||
def debug_memory_relevance(request: Request, query: str = Form(...)):
|
||||
"""Debug which memories would be triggered for a query"""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
|
||||
|
||||
return {
|
||||
"query": query,
|
||||
"total_memories": len(memories),
|
||||
"relevant_count": len(relevant),
|
||||
"relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
|
||||
for m in relevant]
|
||||
}
|
||||
|
||||
@router.post("/add", response_model=Dict[str, Any])
|
||||
async def api_add_memory(
|
||||
request: Request,
|
||||
memory_data: Optional[MemoryAddRequest] = None
|
||||
):
|
||||
"""Add a new memory entry with optional category, source, and session reference."""
|
||||
from src.auth_helpers import require_privilege
|
||||
require_privilege(request, "can_manage_memory")
|
||||
if memory_data is None:
|
||||
form = await request.form()
|
||||
memory_data = MemoryAddRequest(
|
||||
text=form.get("text"),
|
||||
category=form.get("category", "fact"),
|
||||
source=form.get("source", "user"),
|
||||
session_id=form.get("session_id")
|
||||
)
|
||||
|
||||
user = _owner(request)
|
||||
text = (memory_data.text or "").strip()
|
||||
if not text:
|
||||
raise HTTPException(400, "empty memory")
|
||||
user_mem = memory_manager.load(owner=user)
|
||||
if memory_manager.find_duplicates(text, user_mem):
|
||||
return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
|
||||
|
||||
if memory_data.session_id:
|
||||
try:
|
||||
session_obj = session_manager.get_session(memory_data.session_id)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "Session not found")
|
||||
_assert_session_owner(session_obj, user)
|
||||
|
||||
new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
|
||||
if memory_data.session_id:
|
||||
new_entry["session_id"] = memory_data.session_id
|
||||
all_mem = memory_manager.load_all()
|
||||
all_mem.append(new_entry)
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index
|
||||
if memory_vector and memory_vector.healthy:
|
||||
memory_vector.add(new_entry["id"], text)
|
||||
try:
|
||||
from src.event_bus import fire_event
|
||||
fire_event("memory_added", user)
|
||||
except Exception:
|
||||
logger.debug("memory_added event dispatch failed", exc_info=True)
|
||||
return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
|
||||
|
||||
@router.get("")
|
||||
def api_get_memory(request: Request):
|
||||
"""Return all memory entries with their metadata."""
|
||||
user = _owner(request)
|
||||
return {"memory": memory_manager.load(owner=user)}
|
||||
|
||||
@router.post("/search")
|
||||
def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
|
||||
"""Search across all memories with optional filters."""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
|
||||
if session_id:
|
||||
memories = [m for m in memories if m.get("session_id") == session_id]
|
||||
|
||||
if category:
|
||||
memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
|
||||
|
||||
relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
|
||||
|
||||
return {"memories": relevant, "total": len(relevant), "query": query}
|
||||
|
||||
@router.get("/timeline")
|
||||
def memory_timeline(request: Request):
|
||||
"""Get memories in chronological order with source session information."""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
|
||||
|
||||
results = []
|
||||
for memory in sorted_memories:
|
||||
if "timestamp" in memory:
|
||||
try:
|
||||
dt = datetime.fromtimestamp(memory["timestamp"])
|
||||
memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
except (ValueError, OSError, OverflowError):
|
||||
memory["timestamp_str"] = "Unknown"
|
||||
else:
|
||||
memory["timestamp_str"] = "Unknown"
|
||||
|
||||
session_id = memory.get("session_id")
|
||||
if session_id and session_id in session_manager.sessions:
|
||||
try:
|
||||
session = session_manager.get_session(session_id)
|
||||
if session:
|
||||
_assert_session_owner(session, user)
|
||||
memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
|
||||
except KeyError:
|
||||
memory["session_name"] = "Unknown"
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
memory["session_name"] = "Unknown"
|
||||
else:
|
||||
memory["session_name"] = "Unknown"
|
||||
|
||||
results.append(memory)
|
||||
|
||||
return {"timeline": results, "total": len(results)}
|
||||
|
||||
@router.get("/by-session/{session_id}")
|
||||
def get_memory_by_session(request: Request, session_id: str):
|
||||
"""Get all memories associated with a specific session."""
|
||||
user = _owner(request)
|
||||
try:
|
||||
_session_obj = session_manager.get_session(session_id)
|
||||
except KeyError:
|
||||
raise HTTPException(404, f"Session {session_id} not found")
|
||||
_assert_session_owner(_session_obj, user)
|
||||
memories = memory_manager.load(owner=user)
|
||||
session_memories = [m for m in memories if m.get("session_id") == session_id]
|
||||
|
||||
session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
|
||||
|
||||
try:
|
||||
session = session_manager.get_session(session_id)
|
||||
session_name = session.name if session else f"Session {session_id[:6]}"
|
||||
except KeyError:
|
||||
session_name = f"Session {session_id[:6]}"
|
||||
|
||||
for memory in session_memories:
|
||||
memory["session_name"] = session_name
|
||||
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"session_name": session_name,
|
||||
"memory_count": len(session_memories),
|
||||
"memories": session_memories
|
||||
}
|
||||
|
||||
@router.post("/extract")
|
||||
async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
|
||||
"""Analyze a session's chat history and return memory suggestions."""
|
||||
require_user(request)
|
||||
try:
|
||||
sess = session_manager.get_session(session)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "Session not found")
|
||||
_assert_session_owner(sess, _owner(request))
|
||||
|
||||
system_msg = {
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a helpful assistant. Analyze the entire conversation history provided and extract any "
|
||||
"useful factual statements, contacts, addresses, phone numbers, or other information that the user "
|
||||
"might want to remember for future interactions. Return each piece of information as a JSON object "
|
||||
"with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
|
||||
"Only include information that is specific and likely to be useful later."
|
||||
),
|
||||
}
|
||||
messages = [system_msg] + sess.get_context_messages()
|
||||
|
||||
t_url, t_model, t_headers = resolve_task_endpoint(
|
||||
sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
|
||||
)
|
||||
|
||||
try:
|
||||
suggestion_text = await llm_call_async(
|
||||
t_url,
|
||||
t_model,
|
||||
messages,
|
||||
temperature=0.2,
|
||||
max_tokens=500,
|
||||
headers=t_headers,
|
||||
)
|
||||
try:
|
||||
suggestions = json.loads(suggestion_text)
|
||||
if isinstance(suggestions, list):
|
||||
suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
|
||||
else:
|
||||
suggestions = []
|
||||
except json.JSONDecodeError:
|
||||
suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
|
||||
|
||||
return {"suggestions": [s for s in suggestions if s]}
|
||||
except Exception as e:
|
||||
logger.error(f"LLM memory extraction failed (session {session}): {e}")
|
||||
fallback = memory_manager.extract_memory_from_chat(sess.history, session)
|
||||
return {"suggestions": [item["text"] for item in fallback]}
|
||||
|
||||
@router.post("/audit")
|
||||
async def api_audit_memories(request: Request, session: str = Form(None)):
|
||||
"""Deduplicate and consolidate memories via LLM.
|
||||
|
||||
Uses task/utility/default settings through the shared resolver, with
|
||||
the active session as fallback when no task or utility model is set.
|
||||
Returns before and after memory counts.
|
||||
"""
|
||||
user = _owner(request)
|
||||
fallback_url = fallback_model = None
|
||||
fallback_headers = None
|
||||
if session:
|
||||
try:
|
||||
sess = session_manager.get_session(session)
|
||||
_assert_session_owner(sess, user)
|
||||
fallback_url = sess.endpoint_url
|
||||
fallback_model = sess.model
|
||||
fallback_headers = sess.headers
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
endpoint_url, model, headers = resolve_task_endpoint(
|
||||
fallback_url, fallback_model, fallback_headers, owner=user
|
||||
)
|
||||
|
||||
if not endpoint_url or not model:
|
||||
raise HTTPException(400, "No default model configured — set one in Settings")
|
||||
|
||||
result = await audit_memories(
|
||||
memory_manager,
|
||||
memory_vector,
|
||||
endpoint_url,
|
||||
model,
|
||||
headers,
|
||||
owner=user,
|
||||
)
|
||||
|
||||
if "error" in result and "before" not in result:
|
||||
raise HTTPException(502, f"Audit failed: {result['error']}")
|
||||
|
||||
return {
|
||||
"ok": "error" not in result,
|
||||
"before": result.get("before", 0),
|
||||
"after": result.get("after", 0),
|
||||
"removed": result.get("before", 0) - result.get("after", 0),
|
||||
# True when the audit skipped the LLM because nothing changed
|
||||
# since the last tidy. Frontend already says "Already clean"
|
||||
# for removed==0, so this is here for future use / debugging.
|
||||
"already_tidy": bool(result.get("already_tidy")),
|
||||
}
|
||||
|
||||
@router.post("/import")
|
||||
async def import_memories_from_file(
|
||||
request: Request,
|
||||
session: str | None = Form(None),
|
||||
file: UploadFile = File(...)
|
||||
):
|
||||
"""Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
|
||||
from src.auth_helpers import require_privilege
|
||||
require_privilege(request, "can_manage_memory")
|
||||
|
||||
endpoint_url = None
|
||||
model = None
|
||||
headers = {}
|
||||
|
||||
user = _owner(request)
|
||||
|
||||
if session:
|
||||
try:
|
||||
sess = session_manager.get_session(session)
|
||||
_assert_session_owner(sess, user)
|
||||
except KeyError:
|
||||
sess = None
|
||||
except HTTPException as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
sess = None
|
||||
|
||||
if sess is None:
|
||||
logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
|
||||
endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
|
||||
else:
|
||||
endpoint_url, model, headers = resolve_task_endpoint(
|
||||
sess.endpoint_url, sess.model, sess.headers, owner=user
|
||||
)
|
||||
else:
|
||||
endpoint_url, model, headers = resolve_task_endpoint(owner=user)
|
||||
|
||||
if not endpoint_url or not model:
|
||||
raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
|
||||
|
||||
content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
|
||||
filename = file.filename or "upload"
|
||||
_, ext = os.path.splitext(filename.lower())
|
||||
|
||||
allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
|
||||
if ext not in allowed:
|
||||
raise HTTPException(400, f"Unsupported file type: {ext}")
|
||||
|
||||
# Extract text based on file type
|
||||
if ext == ".pdf":
|
||||
from src.document_processor import _process_pdf
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
text = _process_pdf(tmp_path, owner=_owner(request))
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
else:
|
||||
try:
|
||||
text = content.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
from charset_normalizer import detect
|
||||
encoding = (detect(content) or {}).get("encoding") or "utf-8"
|
||||
text = content.decode(encoding, errors="replace")
|
||||
|
||||
if not text.strip():
|
||||
return {"suggestions": [], "message": "No readable content found"}
|
||||
|
||||
# Fast path: a .json upload that already looks like a memories export
|
||||
# (list of {text, category, ...} dicts, or list of strings) round-trips
|
||||
# directly without spending an LLM call to re-extract its own output.
|
||||
# Without this, re-importing a memories.json from another account
|
||||
# ran the file through the extractor, which often re-emitted the
|
||||
# entries as a numbered list (and the numbering leaked into the
|
||||
# `text` field).
|
||||
if ext == ".json":
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
if isinstance(parsed, list) and parsed:
|
||||
direct = []
|
||||
for item in parsed:
|
||||
if isinstance(item, dict) and item.get("text"):
|
||||
direct.append({
|
||||
"text": _strip_list_prefix(str(item["text"])),
|
||||
"category": item.get("category") or "fact",
|
||||
})
|
||||
elif isinstance(item, str) and item.strip():
|
||||
direct.append({
|
||||
"text": _strip_list_prefix(item.strip()),
|
||||
"category": "fact",
|
||||
})
|
||||
if direct:
|
||||
return {"suggestions": direct, "filename": filename}
|
||||
|
||||
# Truncate very long documents
|
||||
if len(text) > 15000:
|
||||
text = text[:15000] + "\n[Truncated]"
|
||||
|
||||
# Send to LLM for memory extraction
|
||||
import_prompt = (
|
||||
"You are a memory extraction assistant. The user uploaded a document. "
|
||||
"Analyze the text below and extract specific, useful facts — things like "
|
||||
"names, preferences, jobs, locations, relationships, opinions, projects, "
|
||||
"goals, contacts, or any other personal details worth remembering.\n\n"
|
||||
"Rules:\n"
|
||||
"- Each fact should be a short, self-contained statement\n"
|
||||
"- Do NOT extract generic knowledge\n"
|
||||
"- Focus on personal, memorable information\n"
|
||||
"- If there are no useful facts, return an empty array\n\n"
|
||||
"Return a JSON array of objects with 'text' and 'category' fields.\n"
|
||||
"Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
|
||||
"Return ONLY valid JSON, no markdown fences."
|
||||
)
|
||||
|
||||
try:
|
||||
raw = await llm_call_async(
|
||||
endpoint_url,
|
||||
model,
|
||||
[
|
||||
{"role": "system", "content": import_prompt},
|
||||
{"role": "user", "content": f"Document: {filename}\n\n{text}"},
|
||||
],
|
||||
temperature=0.2,
|
||||
max_tokens=2000,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
# Parse JSON
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
|
||||
|
||||
suggestions = json.loads(raw)
|
||||
if isinstance(suggestions, list):
|
||||
normalized = []
|
||||
for s in suggestions:
|
||||
if not s:
|
||||
continue
|
||||
if isinstance(s, dict):
|
||||
s = dict(s)
|
||||
if s.get("text"):
|
||||
s["text"] = _strip_list_prefix(str(s["text"]))
|
||||
normalized.append(s)
|
||||
else:
|
||||
normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
|
||||
suggestions = normalized
|
||||
else:
|
||||
suggestions = []
|
||||
|
||||
return {"suggestions": suggestions, "filename": filename}
|
||||
|
||||
except json.JSONDecodeError:
|
||||
# Fallback: split by lines, stripping any "1.", "2)" markdown-list
|
||||
# numbering the model added so saved memories don't keep the prefix.
|
||||
lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
|
||||
return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
|
||||
except Exception as e:
|
||||
logger.error(f"Memory import extraction failed: {e}")
|
||||
raise HTTPException(502, f"LLM extraction failed: {str(e)}")
|
||||
|
||||
@router.post("/{memory_id}/pin")
|
||||
def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
|
||||
"""Pin or unpin a memory. Pinned memories are always included in context."""
|
||||
user = _owner(request)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
all_mem[i]["pinned"] = pinned
|
||||
memory_manager.save(all_mem)
|
||||
return {"ok": True, "pinned": pinned}
|
||||
raise HTTPException(404, f"Memory item {memory_id} not found")
|
||||
|
||||
# Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
|
||||
@router.get("/{memory_id}")
|
||||
def get_memory_item(request: Request, memory_id: str):
|
||||
"""Get a specific memory item by ID."""
|
||||
user = _owner(request)
|
||||
memories = memory_manager.load(owner=user)
|
||||
for memory in memories:
|
||||
if memory["id"] == memory_id:
|
||||
return {"memory": memory}
|
||||
|
||||
raise HTTPException(404, "Memory not found")
|
||||
|
||||
@router.put("/{memory_id}")
|
||||
def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
|
||||
"""Update an existing memory item with new text and optional category."""
|
||||
user = _owner(request)
|
||||
all_mem = memory_manager.load_all()
|
||||
for i, memory in enumerate(all_mem):
|
||||
if memory["id"] == memory_id:
|
||||
_verify_memory_owner(memory, user)
|
||||
all_mem[i]["text"] = text.strip()
|
||||
if category:
|
||||
all_mem[i]["category"] = category
|
||||
all_mem[i]["timestamp"] = int(time.time())
|
||||
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index (remove old, add updated)
|
||||
if memory_vector and memory_vector.healthy:
|
||||
memory_vector.remove(memory_id)
|
||||
memory_vector.add(memory_id, text.strip())
|
||||
return {"ok": True, "message": "Memory updated successfully"}
|
||||
|
||||
raise HTTPException(404, f"Memory item {memory_id} not found")
|
||||
|
||||
@router.delete("/{memory_id}")
|
||||
def delete_memory(request: Request, memory_id: str):
|
||||
"""Delete a memory item by its ID."""
|
||||
user = _owner(request)
|
||||
all_mem = memory_manager.load_all()
|
||||
|
||||
# Find and verify ownership before deleting
|
||||
target = next((m for m in all_mem if m["id"] == memory_id), None)
|
||||
if not target:
|
||||
raise HTTPException(404, f"Memory item {memory_id} not found")
|
||||
_verify_memory_owner(target, user)
|
||||
|
||||
all_mem = [m for m in all_mem if m["id"] != memory_id]
|
||||
memory_manager.save(all_mem)
|
||||
# Sync vector index
|
||||
if memory_vector and memory_vector.healthy:
|
||||
memory_vector.remove(memory_id)
|
||||
return {"ok": True, "message": "Memory deleted successfully"}
|
||||
|
||||
return router
|
||||
_sys.modules[__name__] = _canonical
|
||||
|
||||
+13
-20
@@ -16,6 +16,11 @@ from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
from core.platform_compat import IS_APPLE_SILICON, which_tool
|
||||
from core.middleware import INTERNAL_TOOL_USER
|
||||
from src.host_docker_access import (
|
||||
HOST_DOCKER_ACCESS_HINT,
|
||||
host_docker_access_enabled as _host_docker_access_enabled,
|
||||
running_in_container as _running_in_container,
|
||||
)
|
||||
from src.optional_deps import prepare_optional_dependency_import
|
||||
|
||||
# POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
|
||||
@@ -103,32 +108,17 @@ logger = logging.getLogger(__name__)
|
||||
PTY_SUPPORTED = pty is not None and fcntl is not None and hasattr(os, "setsid")
|
||||
|
||||
|
||||
DOCKER_IN_CONTAINER_HINT = (
|
||||
"Not available inside the Odysseus container by design. The image ships no "
|
||||
"docker CLI and no host socket is mounted. Run Docker-backed launches on a "
|
||||
"remote server, where docker is checked over SSH. Mounting /var/run/docker.sock "
|
||||
"into the container would grant it host-root access, so only do that if you "
|
||||
"accept that risk."
|
||||
)
|
||||
|
||||
|
||||
def _running_in_container(dockerenv_path="/.dockerenv", cgroup_path="/proc/1/cgroup"):
|
||||
if os.path.exists(dockerenv_path):
|
||||
return True
|
||||
try:
|
||||
with open(cgroup_path, "r", encoding="utf-8") as fh:
|
||||
contents = fh.read()
|
||||
except OSError:
|
||||
return False
|
||||
return any(token in contents for token in ("docker", "containerd", "kubepods"))
|
||||
DOCKER_IN_CONTAINER_HINT = HOST_DOCKER_ACCESS_HINT
|
||||
|
||||
|
||||
DockerRowStatus = namedtuple("DockerRowStatus", ["applicable", "install_hint"])
|
||||
PackageUpdateStatus = namedtuple("PackageUpdateStatus", ["available", "note"])
|
||||
|
||||
|
||||
def _docker_row_status(*, on_remote, in_container, installed, default_hint):
|
||||
local_docker_unavailable = not on_remote and in_container and not installed
|
||||
def _docker_row_status(
|
||||
*, on_remote, in_container, installed, default_hint, host_docker_access=False
|
||||
):
|
||||
local_docker_unavailable = not on_remote and in_container and not host_docker_access
|
||||
if local_docker_unavailable:
|
||||
return DockerRowStatus(applicable=False, install_hint=DOCKER_IN_CONTAINER_HINT)
|
||||
return DockerRowStatus(applicable=True, install_hint=default_hint)
|
||||
@@ -1510,6 +1500,9 @@ def setup_shell_routes() -> APIRouter:
|
||||
in_container=_running_in_container() if not on_remote else False,
|
||||
installed=pkg["installed"],
|
||||
default_hint=pkg.get("install_hint"),
|
||||
host_docker_access=(
|
||||
_host_docker_access_enabled() if not on_remote else False
|
||||
),
|
||||
)
|
||||
pkg["applicable"] = status.applicable
|
||||
pkg["install_hint"] = status.install_hint
|
||||
|
||||
Reference in New Issue
Block a user