Merge remote-tracking branch 'origin/dev'

# Conflicts:
#	routes/contacts_routes.py
This commit is contained in:
pewdiepie-archdaemon
2026-07-07 00:51:34 +00:00
80 changed files with 5885 additions and 2383 deletions
+9 -3
View File
@@ -512,7 +512,7 @@ Bulk delete/archive/mark emails. Use this for "delete all those" after listing e
{"action": "create_event", "summary": "<event title>", "dtstart": "<natural language or ISO datetime>"}
```
Calendar event management (CalDAV). Actions: `list_events`, `create_event`, `update_event`, `delete_event`, `list_calendars`. \
For `list_events`: {start?, end?, calendar?}; prefer `start`/`end` for the range, though start_date/end_date and from/to aliases are accepted. \
For `list_events`: {action: "list_events", start: "YYYY-MM-DDT00:00:00", end: "YYYY-MM-DDT00:00:00", calendar?}; resolve month/week phrases yourself from the Current date and time context and do not pass a loose `query` field. Prefer `start`/`end`; start_time/end_time, start_date/end_date, and from/to aliases are accepted. \
For `create_event`: {summary, dtstart, dtend?, duration?, calendar?, location?, description?, reminder_minutes?, rrule?}. \
For `update_event`: {uid, summary?, dtstart?, dtend?, all_day?, location?, description?, event_type?, importance?, rrule?}. Pass `rrule: ""` to remove recurrence and make a repeating event a single event. \
`dtstart` accepts natural language ("tomorrow at 1pm", "in 2 hours", "next monday 9am") or ISO ("2026-05-12T13:00:00"). \
@@ -2845,11 +2845,17 @@ async def stream_agent_loop(
)
logger.info(f"[tool-rag] Retrieved tools for query: {sorted(_relevant_tools - ALWAYS_AVAILABLE)}")
except asyncio.TimeoutError:
# Leave _relevant_tools unset so the keyword fallback
# below still runs. Hard-coding ALWAYS_AVAILABLE here
# skipped the deterministic keyword hints whenever the
# embedding backend was slow (e.g. a remote endpoint
# cold-loading its model), silently stripping email/
# calendar tools from queries that named them outright.
logger.warning(
"[tool-rag] Retrieval exceeded %.1fs; falling back to always-available tools",
"[tool-rag] Retrieval exceeded %.1fs; falling back to keyword tool selection",
_TOOL_SELECTION_TIMEOUT_SECONDS,
)
_relevant_tools = set(ALWAYS_AVAILABLE)
_relevant_tools = None
except Exception as e:
logger.warning(f"[tool-rag] Retrieval failed, using keyword fallback: {e}")
_relevant_tools = None
+27 -4
View File
@@ -281,7 +281,13 @@ class LsTool:
class GlobTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
from src.tool_execution import (
_SENSITIVE_BASENAMES,
_is_sensitive_path,
_resolve_tool_path,
_resolve_search_root,
_truncate,
)
args = {}
_s = (content or "").strip()
if _s.startswith("{"):
@@ -322,7 +328,11 @@ class GlobTool:
) == nbase
except ValueError:
inside = False
if inside and os.path.exists(cand):
# A literal that names a deny-listed sensitive file (.env,
# .ssh/id_rsa, …) falls through to the walk, which skips it —
# otherwise glob would surface secret paths that read_file /
# grep already refuse to touch.
if inside and os.path.exists(cand) and not _is_sensitive_path(cand):
return [cand], None
# Literal not at exact path — fall through to walk so
# e.g. "foo.py" still matches at any depth (like rglob).
@@ -334,11 +344,20 @@ class GlobTool:
for dp, dns, fns in os.walk(base):
# Prune skipped dirs before descending (unlike rglob which
# descends first then filters — fatal on large node_modules).
dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
# Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
# never enumerates the keys/tokens inside them.
dns[:] = [
d for d in dns
if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
]
for name in fns + dns:
full = os.path.join(dp, name)
rel = os.path.relpath(full, base).replace(os.sep, "/")
if regex.fullmatch(rel) or regex.fullmatch(name):
# Skip deny-listed sensitive files (.env, id_rsa,
# known_hosts, …) the same way grep does.
if _is_sensitive_path(os.path.realpath(full)):
continue
try:
mtime = os.stat(full).st_mtime
except OSError:
@@ -405,8 +424,12 @@ class GrepTool:
cmd.append("--ignore-case")
if glob_pat:
cmd += ["--glob", glob_pat]
# --iglob (not --glob) so the exclusion is case-insensitive:
# on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
# resolve to the same secret as their lowercase forms, and the
# Python fallback below already folds case via _is_sensitive_path.
for _pat in _SENSITIVE_FILE_PATTERNS:
cmd += ["--glob", f"!*{_pat}*"]
cmd += ["--iglob", f"!*{_pat}*"]
for _d in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{_d}/**"]
cmd += ["--regexp", pattern, root]
+6 -2
View File
@@ -184,8 +184,12 @@ async def send_to_session(content: str, session_id: Optional[str] = None, owner:
if not sess:
return {"error": f"Session '{target_sid}' not found"}
# Owner-scope: reject access to another user's session
if owner and getattr(sess, "owner", None) and sess.owner != owner:
# Owner-scope: reject access to another user's session. When the caller is
# authenticated, a null-owner (legacy / auth-was-off) session is not theirs
# either — list_sessions (get_sessions_for_user) and manage_session already
# exclude those, so treating it as reachable here let an authenticated agent
# read/write a session the other tools hide. Require an exact owner match.
if owner and getattr(sess, "owner", None) != owner:
return {"error": f"Session '{target_sid}' not found"}
if not message:
+24 -1
View File
@@ -216,7 +216,14 @@ def _normalize_integration_base_url(base_url: Any) -> str:
def _join_integration_url(base_url: str, path: str) -> str:
return urljoin(base_url.rstrip("/") + "/", path.lstrip("/"))
base = base_url.rstrip("/")
rel = path.lstrip("/")
if not rel:
# A bare "/" must resolve to the base URL itself, not base + "/".
# POST-to-base integrations (e.g. Discord webhooks) 404 on the
# trailing-slash variant of their URL.
return base
return urljoin(base + "/", rel)
def load_integrations() -> List[Dict[str, Any]]:
@@ -394,6 +401,22 @@ async def execute_api_call(
return {"error": "Path must not contain a fragment", "exit_code": 1}
url = _join_integration_url(base_url, path)
# SSRF guard — same check used by the gallery endpoint, embeddings,
# CardDAV, and the reminder webhook sender. Link-local / metadata
# addresses (169.254.x.x — the cloud credential-exfil vector) are always
# rejected; INTEGRATION_API_BLOCK_PRIVATE_IPS=true also blocks RFC-1918 /
# loopback for locked-down deployments. Private stays allowed by default
# because LAN integrations (Home Assistant, Miniflux, ntfy) are the
# primary use case.
from src.url_safety import check_outbound_url
block_private = os.getenv(
"INTEGRATION_API_BLOCK_PRIVATE_IPS", "false"
).lower() == "true"
ok, reason = check_outbound_url(url, block_private=block_private)
if not ok:
return {"error": f"URL rejected: {reason}", "exit_code": 1}
method = method.upper()
# Build headers
+2 -2
View File
@@ -32,9 +32,9 @@ class RAGManager:
logger.info("RAGManager initialized as wrapper for VectorRAG")
# Delegate all methods to VectorRAG
def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
def search(self, query: str, k: int = 5, owner: Optional[str] = None) -> List[Dict[str, Any]]:
"""Search for documents - delegates to VectorRAG."""
return self.vector_rag.search(query, k)
return self.vector_rag.search(query, k, owner=owner)
def index_personal_documents(
self,
+47
View File
@@ -0,0 +1,47 @@
"""Shared privilege policy for scheduled task actions."""
from __future__ import annotations
ADMIN_ONLY_TASK_ACTIONS = frozenset({
"run_local",
"run_script",
"ssh_command",
"cookbook_serve",
})
def is_admin_only_task_action(task_type: str | None, action: str | None) -> bool:
return (task_type or "llm") == "action" and (action or "") in ADMIN_ONLY_TASK_ACTIONS
def owner_has_admin_task_privileges(owner: str | None) -> bool:
try:
from src.auth_helpers import _auth_disabled
if _auth_disabled():
return True
except Exception:
pass
if owner:
try:
from core.middleware import INTERNAL_TOOL_USER
if owner == INTERNAL_TOOL_USER:
return True
except Exception:
pass
try:
from core.auth import AuthManager
auth = AuthManager()
if not auth.is_configured:
return True
if not owner:
return False
return bool(auth.is_admin(owner))
except Exception:
pass
if not owner:
return False
return False
+26
View File
@@ -10,6 +10,10 @@ from datetime import datetime, timedelta, timezone
from typing import Any, Awaitable, Callable, Dict, Tuple
from core.auth import RESERVED_USERNAMES
from src.task_action_policy import (
is_admin_only_task_action,
owner_has_admin_task_privileges,
)
logger = logging.getLogger(__name__)
@@ -821,6 +825,28 @@ class TaskScheduler:
db.commit()
return
if (
is_admin_only_task_action(task.task_type, task.action)
and not owner_has_admin_task_privileges(task.owner)
):
msg = f"Action '{task.action}' requires admin privileges"
blocked = db.query(TaskRun).filter(TaskRun.id == run_id).first()
if blocked:
blocked.status = "error"
blocked.result = msg
blocked.error = msg
blocked.finished_at = _utcnow()
task.status = "paused"
task.next_run = None
task.last_run = _utcnow()
logger.warning(
"Paused admin-only task %s for non-admin owner %r",
task_id,
task.owner,
)
db.commit()
return
if gate_foreground:
waiting = db.query(TaskRun).filter(TaskRun.id == run_id).first()
if waiting and waiting.status == "queued":
+18 -8
View File
@@ -71,25 +71,35 @@ _SENSITIVE_FILE_PATTERNS: tuple[str, ...] = (
"known_hosts",
)
# Case-folded views used for matching. On a case-insensitive filesystem
# (Windows, default macOS) ".SSH/AUTHORIZED_KEYS" and ".env" resolve to the
# same protected files as their lowercase forms, so the deny-list has to fold
# case before comparing — the sibling resolver already normcases paths for the
# same reason. casefold (not os.path.normcase) because normcase is a no-op on
# POSIX, which is exactly where the macOS read-exfil path lives.
_SENSITIVE_BASENAMES_CF: frozenset[str] = frozenset(b.casefold() for b in _SENSITIVE_BASENAMES)
_SENSITIVE_FILE_PATTERNS_CF: frozenset[str] = frozenset(p.casefold() for p in _SENSITIVE_FILE_PATTERNS)
def _is_sensitive_path(resolved: str) -> bool:
"""Return True if *resolved* falls under a sensitive directory or
matches a sensitive filename — regardless of what root it sits under.
Matching is case-insensitive: on Windows / default macOS a case-variant
name (``.SSH``, ``AUTHORIZED_KEYS``, ``Id_Rsa``) points at the same file as
the lowercase form, so a case-sensitive check would let it slip past the
deny-list in every file tool that relies on it.
"""
parts = resolved.split(os.sep)
filenames: set[str] = {parts[-1]} if parts else set()
parts = [p.casefold() for p in resolved.split(os.sep)]
filename = parts[-1] if parts else ""
# Check if any path component is a sensitive directory.
for part in parts:
if part in _SENSITIVE_BASENAMES:
if part in _SENSITIVE_BASENAMES_CF:
return True
# Check filename against known sensitive files.
for pat in _SENSITIVE_FILE_PATTERNS:
if pat in filenames:
return True
return False
return filename in _SENSITIVE_FILE_PATTERNS_CF
def _tool_path_roots() -> list[str]:
+2 -2
View File
@@ -565,8 +565,8 @@ FUNCTION_TOOL_SCHEMAS = [
"uid": {"type": "string", "description": "Event UID (for update/delete)"},
"calendar_href": {"type": "string", "description": "Specific calendar URL (optional; defaults to first calendar)"},
"calendar": {"type": "string", "description": "Filter list_events by calendar name or href"},
"start": {"type": "string", "description": "list_events range start (ISO datetime); defaults to today. Prefer start; backend also accepts start_date, range_start, from, dtstart, since."},
"end": {"type": "string", "description": "list_events range end (ISO datetime); defaults to +14 days. Prefer end; backend also accepts end_date, range_end, to, dtend, until."},
"start": {"type": "string", "description": "list_events range start (ISO datetime). Use this for month/week requests after resolving the date range; do not pass a loose query string. Prefer start; backend also accepts start_time, start_date, range_start, from, dtstart, since."},
"end": {"type": "string", "description": "list_events range end (ISO datetime). Use this for month/week requests after resolving the date range; defaults to +14 days only when no range is requested. Prefer end; backend also accepts end_time, end_date, range_end, to, dtend, until."},
"event_type": {"type": "string", "description": "Tag / category for the event. Common values: work, personal, health, travel, meal, social, admin, other. Aliases accepted: tag, category, type."},
"importance": {"type": "string", "enum": ["low", "normal", "high", "critical"], "description": "Priority level (defaults to 'normal')"},
"reminder_minutes": {"type": "integer", "description": "For create_event: create an Odysseus reminder this many minutes before the event, e.g. 5 for 'reminder 5 min before'."},
+2
View File
@@ -54,6 +54,8 @@ def _parse_tool_args(content):
if isinstance(content, str):
try:
args = json.loads(content) if content.strip() else {}
if not isinstance(args, dict):
args = {}
except (json.JSONDecodeError, TypeError) as e:
raise ValueError(str(e))
elif isinstance(content, dict):
+11 -2
View File
@@ -208,11 +208,20 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
elif action == "list_events":
try:
start_raw = _first_nonempty_arg(
"start", "start_date", "range_start", "from", "dtstart", "since"
"start", "start_time", "start_date", "range_start", "from", "dtstart", "since"
)
end_raw = _first_nonempty_arg(
"end", "end_date", "range_end", "to", "dtend", "until"
"end", "end_time", "end_date", "range_end", "to", "dtend", "until"
)
query_raw = args.get("query") or args.get("date_range") or args.get("range")
if query_raw and (not start_raw or not end_raw):
return {
"error": (
"list_events needs explicit start/end ISO datetimes; "
f"resolve the requested range ({query_raw!r}) and call manage_calendar again."
),
"exit_code": 1,
}
if start_raw:
start_dt = _parse_dt(start_raw)
else:
+152 -5
View File
@@ -7,10 +7,12 @@ import ipaddress
import json
import logging
import re
import ssl
from datetime import datetime, timezone
from typing import Optional
from urllib.parse import urlparse
import httpcore
import httpx
from src.database import SessionLocal, Webhook
@@ -125,6 +127,128 @@ def validate_webhook_url(url: str) -> str:
return url
def _validated_public_ips(url: str) -> list:
"""Resolve *url*'s host and return its IPs, raising ValueError if any is
private/internal.
``validate_webhook_url`` resolves the host to decide accept/reject, but the
subsequent ``httpx`` connect re-resolves independently — so a DNS record
that flips between the two lookups (rebinding) can slip an internal IP past
the check. Callers pin the delivery connection to the IP this function
returns, closing that TOCTOU. Fail closed: an unresolvable or partly-private
result raises rather than returning a usable IP.
"""
parsed = urlparse(url)
hostname = (parsed.hostname or "").strip()
if not hostname:
raise ValueError("URL must have a hostname")
try:
literal = ipaddress.ip_address(hostname)
except ValueError:
literal = None
if literal is not None:
if _ip_is_private(literal):
raise ValueError("URL must not point to private/internal addresses")
return [literal]
addrs = _resolve_hostname_ips(hostname)
if not addrs or any(_ip_is_private(a) for a in addrs):
raise ValueError("URL must not point to private/internal addresses")
return addrs
# httpcore raises its own exception hierarchy; map the ones a simple POST can
# surface back to their httpx equivalents so callers' `except httpx.*` blocks
# (and sanitize_error) behave exactly as they did with the default transport.
_HTTPCORE_TO_HTTPX_EXC = {
httpcore.ConnectError: httpx.ConnectError,
httpcore.ConnectTimeout: httpx.ConnectTimeout,
httpcore.NetworkError: httpx.NetworkError,
httpcore.PoolTimeout: httpx.PoolTimeout,
httpcore.ProtocolError: httpx.ProtocolError,
httpcore.ReadError: httpx.ReadError,
httpcore.ReadTimeout: httpx.ReadTimeout,
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
httpcore.TimeoutException: httpx.TimeoutException,
httpcore.WriteError: httpx.WriteError,
httpcore.WriteTimeout: httpx.WriteTimeout,
}
class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend):
"""Async network backend that routes every TCP connect to a fixed IP.
httpcore derives TLS SNI and the ``Host`` header from the request URL, not
from the connect host, so pinning only the socket destination keeps
certificate validation and vhost routing pointed at the original hostname.
"""
def __init__(self, ip: ipaddress._BaseAddress):
self._ip = str(ip)
self._real = httpcore.AnyIOBackend()
async def connect_tcp(self, host, port, timeout=None, local_address=None,
socket_options=None):
return await self._real.connect_tcp(
self._ip, port, timeout, local_address, socket_options
)
async def connect_unix_socket(self, path, timeout=None, socket_options=None):
return await self._real.connect_unix_socket(path, timeout, socket_options)
async def sleep(self, seconds: float) -> None:
return await self._real.sleep(seconds)
class _PinnedAsyncTransport(httpx.AsyncBaseTransport):
"""httpx transport that pins the TCP connect to a pre-resolved public IP.
Uses only public ``httpcore`` / ``httpx`` APIs. The request URL is passed
through unchanged (Host + SNI stay the original hostname); only the socket
destination is pinned, closing the DNS-rebinding TOCTOU between the SSRF
check and the connect. HTTP/1.1 only — webhook deliveries are small POSTs.
"""
def __init__(self, ip: ipaddress._BaseAddress):
self._pool = httpcore.AsyncConnectionPool(
ssl_context=ssl.create_default_context(),
http1=True,
http2=False,
network_backend=_PinnedAsyncBackend(ip),
)
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
core_req = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
try:
core_resp = await self._pool.handle_async_request(core_req)
content = b"".join([chunk async for chunk in core_resp.aiter_stream()])
await core_resp.aclose()
except Exception as exc:
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
if mapped is not None:
raise mapped(str(exc)) from exc
raise
return httpx.Response(
status_code=core_resp.status,
headers=core_resp.headers,
content=content,
extensions=core_resp.extensions,
)
async def aclose(self) -> None:
await self._pool.aclose()
def validate_events(events_str: str) -> str:
"""Validate comma-separated event names. Returns cleaned string."""
events = [e.strip() for e in events_str.split(",") if e.strip()]
@@ -198,8 +322,11 @@ def sanitize_error(error: str, max_len: int = 200) -> str:
class WebhookManager:
def __init__(self, api_key_manager=None):
# Disable redirects to prevent SSRF via redirect chains
self._client = httpx.AsyncClient(timeout=10, follow_redirects=False)
# No shared client: each delivery builds a short-lived client whose
# transport is pinned to the SSRF-approved IP (see _deliver /
# _send_request), so a single reusable client can't be pointed at
# different pinned hosts. Redirects stay disabled on every delivery
# client to prevent SSRF via redirect chains.
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._api_key_manager = api_key_manager
# Strong references to in-flight fire-and-forget tasks. asyncio only
@@ -262,11 +389,28 @@ class WebhookManager:
decrypted = self._decrypt_secret(encrypted_secret)
await self._deliver(webhook_id, url, decrypted, "webhook.test", {"message": "Test ping from Odysseus"})
async def _send_request(self, url: str, body: str, headers: dict,
ip: ipaddress._BaseAddress) -> httpx.Response:
"""POST *body* to *url* with the TCP connect pinned to *ip*.
Overridable seam: tests replace this to avoid real sockets. Redirects
are disabled so a 3xx can't bounce the delivery to another host.
"""
transport = _PinnedAsyncTransport(ip)
async with httpx.AsyncClient(
timeout=10, follow_redirects=False, transport=transport,
) as client:
return await client.post(url, content=body, headers=headers)
async def _deliver(self, webhook_id: str, url: str, secret: Optional[str], event: str, payload: dict):
"""Internal delivery. Never call directly from outside this class (use deliver_test)."""
# Re-validate URL at delivery time in case DB was tampered with
# Re-validate URL at delivery time in case DB was tampered with, and
# capture the exact IPs that passed the check so the connect can be
# pinned to one of them (closes the DNS-rebinding TOCTOU: the check
# below and the socket connect no longer resolve independently).
try:
validate_webhook_url(url)
pinned_ips = _validated_public_ips(url)
except ValueError as e:
logger.warning(f"Webhook {webhook_id} has invalid URL, skipping: {e}")
return
@@ -283,7 +427,7 @@ class WebhookManager:
db = SessionLocal()
try:
resp = await self._client.post(url, content=body, headers=headers)
resp = await self._send_request(url, body, headers, pinned_ips[0])
db.query(Webhook).filter(Webhook.id == webhook_id).update({
"last_triggered_at": _utcnow(),
"last_status_code": resp.status_code,
@@ -305,4 +449,7 @@ class WebhookManager:
db.close()
async def close(self):
await self._client.aclose()
# Delivery clients are per-request and closed via their async context
# manager, so there is no long-lived client to tear down here. Kept for
# API compatibility with callers (e.g. app shutdown).
return None