4 Commits

Author SHA1 Message Date
Moniz 8c943226f8 fix(mobile): stack the model-comparison grid into one column on phones (#4979)
The comparison grid hard-codes 2-4 equal columns with no phone breakpoint, so at
390px two models get ~178px columns and four get ~88px columns. Each column is a
full scrolling chat, so content is unreadably over-wrapped and clipped. On
phones (<=768px), stack the panes into a single scrollable column. Desktop is
unaffected.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-02 17:28:23 +02:00
holden093 0dc98ec9b9 fix(ui): prevent race condition in default chat model dropdown init (#5024)
Setting epSel.value triggered an async change event whose handler
called refreshModels('') — wiping the correct model selection that
refreshModels(settings.default_model) had just applied moments earlier.
The dropdown silently fell back to the alphabetically-first model
(deepseek-v4-flash instead of qwen-3.6-35B-A3B).

Moved the change listener registration to after the settings block
so the async change event fires before any listener exists. The
utility and teacher sections already followed this pattern.
2026-07-02 17:05:55 +02:00
Ernest Hysa 5e9b415bd9 fix(search): pin httpx connection to resolved IP to block DNS rebinding (#704)
* fix(search): pin DNS-validated fetch connections

Rebase the DNS-rebinding SSRF fix onto current dev after search content moved behind the services.search.content canonical module.

Integrate the pinned httpcore NetworkBackend/BaseTransport approach with the current size-capped Client.stream fetch path, preserving Host/SNI semantics while forcing TCP connect to the already validated resolved IP.

Keep src.search.content as the compatibility wrapper and preserve existing OG-image http(s) behavior; this avoids reintroducing the unrelated scope changes that previously blocked review.

Add the explicit httpcore>=1.0,<2.0 requirement used by the public httpcore NetworkBackend and ConnectionPool APIs.

* test(search): restore and rebase DNS rebinding regressions

Keep the current security regression coverage that the stale PR branch had deleted, including auth-disabled localhost bypass and Ollama cookbook hardening tests.

Carry forward the DNS-rebinding coverage for private resolve blocking, pinned TCP connect behavior, Host header preservation, redirect revalidation, and the BaseTransport/public-httpcore static guard.

Update redirect tests to mock the current Client.stream-based capped fetch path rather than the older httpx.stream/get path.

* test(search): adapt size-cap fetch tests to pinned client stream

The DNS-rebinding repair moved _get_public_url from the module-level httpx.stream shortcut to httpx.Client(...).stream(...) so the fetch can use the pinned transport.

Keep the existing size-cap test fakes by routing Client.stream through the monkeypatched httpx.stream only when a test has installed that fake; otherwise fall back to a real Client.

This fixes the CI failures in tests/test_web_fetch_size_caps.py without touching unrelated upload-handler atomicity behavior, which is already flaky on clean origin/dev.

---------

Co-authored-by: Alexandre Teixeira <alexandremagteixeira@gmail.com>
2026-07-02 13:08:06 +01:00
Alexandre Teixeira b1f9f67d9d fix(security): confine research file paths (#4986) 2026-07-02 11:58:35 +01:00
8 changed files with 869 additions and 78 deletions
+1
View File
@@ -3,6 +3,7 @@ uvicorn
python-multipart python-multipart
python-dotenv python-dotenv
httpx httpx
httpcore>=1.0,<2.0
pydantic>=2.13.4 pydantic>=2.13.4
pydantic-settings>=2.14.1 pydantic-settings>=2.14.1
SQLAlchemy SQLAlchemy
+30 -8
View File
@@ -20,6 +20,26 @@ from src.constants import DEEP_RESEARCH_DIR
_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
def _confine_research_path(session_id: str) -> Path:
"""Return the resolved Path for session_id's JSON inside DEEP_RESEARCH_DIR.
Validates the session ID format and asserts containment after symlink
expansion. Raises HTTPException(400) on format failures, traversal
attempts, absolute-path injection, and symlink escape so every caller
gets a safe, confined path with no extra validation needed.
"""
if not _SESSION_ID_RE.fullmatch(session_id):
raise HTTPException(400, "Invalid session ID")
root = Path(DEEP_RESEARCH_DIR).resolve()
candidate = (root / f"{session_id}.json").resolve()
try:
candidate.relative_to(root)
except ValueError:
raise HTTPException(400, "Invalid session ID")
return candidate
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Model-name substrings that are NOT chat/generation models — research must # Model-name substrings that are NOT chat/generation models — research must
@@ -183,7 +203,10 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
if entry is not None: if entry is not None:
return entry.get("owner", "") == user return entry.get("owner", "") == user
# Task no longer in memory — check the persisted JSON. # Task no longer in memory — check the persisted JSON.
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" try:
path = _confine_research_path(session_id)
except HTTPException:
return False
if not path.exists(): if not path.exists():
return False return False
try: try:
@@ -247,7 +270,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
def _assert_owns_research(session_id: str, user: str) -> None: def _assert_owns_research(session_id: str, user: str) -> None:
"""404-not-403 ownership gate for a research session's on-disk JSON. """404-not-403 ownership gate for a research session's on-disk JSON.
Use BEFORE returning any data or mutating the file.""" Use BEFORE returning any data or mutating the file."""
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" path = _confine_research_path(session_id)
if not path.exists(): if not path.exists():
raise HTTPException(404, "Research not found") raise HTTPException(404, "Research not found")
try: try:
@@ -361,7 +384,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
summary, stats — used by the Library preview panel.""" summary, stats — used by the Library preview panel."""
user = _require_user(request) user = _require_user(request)
_validate_session_id(session_id) _validate_session_id(session_id)
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" path = _confine_research_path(session_id)
if not path.exists(): if not path.exists():
raise HTTPException(404, "Research not found") raise HTTPException(404, "Research not found")
try: try:
@@ -378,7 +401,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
"""Soft-archive / restore a research report (sets `archived` in its JSON).""" """Soft-archive / restore a research report (sets `archived` in its JSON)."""
user = _require_user(request) user = _require_user(request)
_validate_session_id(session_id) _validate_session_id(session_id)
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" path = _confine_research_path(session_id)
if not path.exists(): if not path.exists():
raise HTTPException(404, "Research not found") raise HTTPException(404, "Research not found")
try: try:
@@ -398,8 +421,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
"""Delete a research result from disk.""" """Delete a research result from disk."""
user = _require_user(request) user = _require_user(request)
_validate_session_id(session_id) _validate_session_id(session_id)
data_dir = Path(DEEP_RESEARCH_DIR) json_path = _confine_research_path(session_id)
json_path = data_dir / f"{session_id}.json"
deleted = False deleted = False
if json_path.exists(): if json_path.exists():
# SECURITY: verify ownership before letting the caller delete it. # SECURITY: verify ownership before letting the caller delete it.
@@ -561,7 +583,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
raise HTTPException(404, "No research found for this session") raise HTTPException(404, "No research found for this session")
result = research_handler.get_result(session_id) result = research_handler.get_result(session_id)
if result is None: if result is None:
p = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" p = _confine_research_path(session_id)
if p.exists(): if p.exists():
d = json.loads(p.read_text(encoding="utf-8")) d = json.loads(p.read_text(encoding="utf-8"))
return { return {
@@ -601,7 +623,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
sources = research_handler.get_sources(session_id) or [] sources = research_handler.get_sources(session_id) or []
query = "" query = ""
path = Path(DEEP_RESEARCH_DIR) / f"{session_id}.json" path = _confine_research_path(session_id)
if path.exists(): if path.exists():
try: try:
disk = json.loads(path.read_text(encoding="utf-8")) disk = json.loads(path.read_text(encoding="utf-8"))
+206 -62
View File
@@ -8,11 +8,13 @@ import os
import re import re
import logging import logging
import socket import socket
import ssl
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import List from typing import Iterable, List, cast
from urllib.parse import urljoin, urlparse from urllib.parse import urljoin, urlparse
import httpx import httpx
import httpcore
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
@@ -91,6 +93,148 @@ def _public_http_url(url: str) -> bool:
return False return False
def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
parsed = urlparse(url)
if parsed.scheme not in ("http", "https") or not parsed.hostname:
raise httpx.RequestError(f"Blocked non-public URL: {url}")
host = (parsed.hostname or "").strip().lower()
if host in ("localhost", "metadata", "metadata.google.internal"):
raise httpx.RequestError(f"Blocked non-public hostname: {host}")
try:
ip = ipaddress.ip_address(host)
if _is_private_address(ip):
raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
return [ip]
except httpx.RequestError:
raise
except ValueError:
pass
addrs = _resolve_hostname_ips(host)
if not addrs or any(_is_private_address(a) for a in addrs):
raise httpx.RequestError(f"Blocked non-public URL: {url}")
return addrs
class _PinnedBackend(httpcore.NetworkBackend):
"""Network backend that connects to a pre-resolved IP.
httpcore derives the TLS SNI and the ``Host`` header from the URL's
origin, not from the host argument passed to ``connect_tcp``. So
routing the TCP connect to a resolved IP while leaving the URL
untouched keeps SNI / vhost behaviour correct and closes the
DNS-rebinding TOCTOU between the SSRF check and the connect.
"""
def __init__(self, ip: ipaddress._BaseAddress):
self._ip = str(ip)
self._real = httpcore.SyncBackend()
def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options=None,
):
return self._real.connect_tcp(
self._ip, port, timeout, local_address, socket_options
)
def connect_unix_socket(self, path, timeout=None, socket_options=None):
return self._real.connect_unix_socket(path, timeout, socket_options)
def sleep(self, seconds: float) -> None:
return self._real.sleep(seconds)
# Map httpcore exception classes to their httpx equivalents. Built
# once at import time from the public exception classes; avoids any
# import of httpx's private transport machinery. httpcore's
# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
# close and retry on its own) — we never expect to see it surface to
# a transport caller, so it has no httpx counterpart here.
_HTTPCORE_TO_HTTPX_EXC = {
httpcore.ConnectError: httpx.ConnectError,
httpcore.ConnectTimeout: httpx.ConnectTimeout,
httpcore.LocalProtocolError: httpx.LocalProtocolError,
httpcore.NetworkError: httpx.NetworkError,
httpcore.PoolTimeout: httpx.PoolTimeout,
httpcore.ProtocolError: httpx.ProtocolError,
httpcore.ProxyError: httpx.ProxyError,
httpcore.ReadError: httpx.ReadError,
httpcore.ReadTimeout: httpx.ReadTimeout,
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
httpcore.TimeoutException: httpx.TimeoutException,
httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
httpcore.WriteError: httpx.WriteError,
httpcore.WriteTimeout: httpx.WriteTimeout,
}
class _PinnedTransport(httpx.BaseTransport):
"""Transport that pins every TCP connect to a pre-resolved IP.
Uses only the public ``httpcore`` and ``httpx`` APIs — no
subclassing of ``httpx.HTTPTransport``, no reads of private
``httpcore.ConnectionPool`` attributes, no imports from
``httpx private transport internals``. The URL is passed through unchanged so SNI
/ vhost work as if httpx had been given the hostname directly;
only the TCP destination is pinned, closing the DNS-rebinding
TOCTOU between the SSRF check and the connect.
"""
def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
self._pool = httpcore.ConnectionPool(
ssl_context=ssl.create_default_context(),
http1=True,
http2=http2,
network_backend=_PinnedBackend(ip),
)
def __enter__(self):
self._pool.__enter__()
return self
def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
self._pool.__exit__(exc_type, exc_value, traceback)
def handle_request(self, request: httpx.Request) -> httpx.Response:
httpcore_req = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
try:
httpcore_resp = self._pool.handle_request(httpcore_req)
# Eager materialisation matches the original
# ``response.text`` usage in fetch_webpage_content. The
# sync pool's stream is a plain Iterable[bytes] despite
# the httpcore type hint unioning the async variant.
content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
except Exception as exc:
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
if mapped is not None:
raise mapped(str(exc)) from exc
raise
return httpx.Response(
status_code=httpcore_resp.status,
headers=httpcore_resp.headers,
content=content,
extensions=httpcore_resp.extensions,
)
def close(self) -> None:
self._pool.close()
class BodyTooLargeError(Exception): class BodyTooLargeError(Exception):
"""The server declared a body larger than the hard fetch ceiling.""" """The server declared a body larger than the hard fetch ceiling."""
@@ -141,78 +285,78 @@ class _CappedFetch:
def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5, def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
max_bytes: int = None) -> "_CappedFetch": max_bytes: int = None) -> "_CappedFetch":
"""Capped streaming GET with SSRF-guarded manual redirects. """Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
The body is streamed and buffering stops at ``max_bytes`` (default: the Each hop is resolved once, validated as public, and then the actual TCP
soft cap), so an oversized resource cannot be pulled into memory or the connection is pinned to that resolved IP. The request URL is left unchanged
content cache in full. When Content-Length already declares a body over so Host and TLS SNI keep the original hostname.
the hard ceiling, the fetch is refused before any body bytes are read.
""" """
cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES) cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
current = url current = url
for _ in range(max_redirects + 1): for _ in range(max_redirects + 1):
if not _public_http_url(current): ips = _resolve_public_ips(current)
raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
# Force identity transfer-encoding. With gzip/deflate the wire bytes # Force identity transfer-encoding. With gzip/deflate the wire bytes
# (and Content-Length) can be a small fraction of the decoded body, so # and Content-Length can be a small fraction of the decoded body, so a
# a tiny compressed response could pass the hard-cap preflight and then # tiny compressed response could pass the hard-cap preflight and then
# expand past the ceiling in a single decoded chunk before the streamed # expand past the ceiling in one decoded chunk before the streamed cap
# cap below can slice it. Identity makes Content-Length the true body # below can slice it.
# size and keeps each streamed chunk bounded by the network read.
req_headers = dict(headers or {}) req_headers = dict(headers or {})
req_headers["Accept-Encoding"] = "identity" req_headers["Accept-Encoding"] = "identity"
with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
follow_redirects=False) as response:
if response.status_code in (301, 302, 303, 307, 308):
location = response.headers.get("location")
if not location:
return _CappedFetch(response.status_code, response.headers, b"",
False, None, response.encoding, str(response.url))
current = urljoin(str(response.url), location)
continue
# A server can ignore the identity request and still return a with httpx.Client(
# compressed body; httpx.iter_bytes would then decode it, and a tiny headers=req_headers,
# gzip can balloon into one decoded chunk far past the cap before we timeout=timeout,
# slice. Refuse a compressed Content-Encoding so the streamed cap follow_redirects=False,
# stays a real memory bound (Content-Length is the compressed wire transport=_PinnedTransport(ips[0]),
# length here, so the preflight and size metadata are unreliable too). ) as client:
enc = (response.headers.get("content-encoding") or "").strip().lower() with client.stream("GET", current) as response:
if enc and enc != "identity": if response.status_code in (301, 302, 303, 307, 308):
raise httpx.RequestError( location = response.headers.get("location")
f"Refusing compressed response (Content-Encoding: {enc}) after " if not location:
"requesting identity: cannot bound decoded body size", return _CappedFetch(response.status_code, response.headers, b"",
request=httpx.Request("GET", current), False, None, response.encoding, str(response.url))
) current = urljoin(str(response.url), location)
continue
declared = None # A server can ignore the identity request and still return a
raw_len = response.headers.get("content-length") # compressed body; httpx.iter_bytes would then decode it, and a
if raw_len and raw_len.isdigit(): # tiny gzip can balloon into one decoded chunk far past the cap.
declared = int(raw_len) # Refuse compressed Content-Encoding so the streamed cap stays
# Refuse before buffering anything when the server already tells # a real memory bound.
# us the body exceeds the absolute ceiling (Content-Length is wire enc = (response.headers.get("content-encoding") or "").strip().lower()
# bytes; the decompressed body can only be larger). if enc and enc != "identity":
if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES: raise httpx.RequestError(
raise BodyTooLargeError(current, declared) f"Refusing compressed response (Content-Encoding: {enc}) after "
"requesting identity: cannot bound decoded body size",
request=httpx.Request("GET", current),
)
declared = None
raw_len = response.headers.get("content-length")
if raw_len and raw_len.isdigit():
declared = int(raw_len)
if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
raise BodyTooLargeError(current, declared)
chunks = []
read = 0
truncated = False
for chunk in response.iter_bytes():
read += len(chunk)
if read > cap:
keep = cap - (read - len(chunk))
if keep > 0:
chunks.append(chunk[:keep])
truncated = True
break
chunks.append(chunk)
return _CappedFetch(response.status_code, response.headers,
b"".join(chunks), truncated, declared,
response.encoding, str(response.url))
chunks = []
read = 0
truncated = False
# We requested identity above, so iter_bytes yields the raw body in
# network-read-sized chunks (no decompression expansion); the cap
# therefore bounds what we actually buffer.
for chunk in response.iter_bytes():
read += len(chunk)
if read > cap:
keep = cap - (read - len(chunk))
if keep > 0:
chunks.append(chunk[:keep])
truncated = True
break
chunks.append(chunk)
return _CappedFetch(response.status_code, response.headers,
b"".join(chunks), truncated, declared,
response.encoding, str(response.url))
raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current)) raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
# PDF extraction (optional dependency) # PDF extraction (optional dependency)
+3 -2
View File
@@ -542,6 +542,9 @@ async function initDefaultChat() {
renderFallbacks(); renderFallbacks();
} catch (e) { console.warn('Failed to load default chat settings', e); } } catch (e) { console.warn('Failed to load default chat settings', e); }
epSel.addEventListener('change', function() { refreshModels(''); saveDefault(); });
modelSel.addEventListener('change', saveDefault);
async function saveDefault() { async function saveDefault() {
try { try {
var clean = _fallbacks.filter(function(f) { return f.endpoint_id && f.model; }); var clean = _fallbacks.filter(function(f) { return f.endpoint_id && f.model; });
@@ -558,8 +561,6 @@ async function initDefaultChat() {
} catch (e) { msg.textContent = 'Failed to save'; msg.style.color = 'var(--red)'; } } catch (e) { msg.textContent = 'Failed to save'; msg.style.color = 'var(--red)'; }
} }
epSel.addEventListener('change', function() { refreshModels(''); saveDefault(); });
modelSel.addEventListener('change', saveDefault);
if (addFbBtn) addFbBtn.addEventListener('click', function() { if (addFbBtn) addFbBtn.addEventListener('click', function() {
var first = enabledEndpoints()[0]; var first = enabledEndpoints()[0];
_fallbacks.push({ endpoint_id: first ? first.id : '', model: '' }); _fallbacks.push({ endpoint_id: first ? first.id : '', model: '' });
+10
View File
@@ -39695,3 +39695,13 @@ body.theme-frosted .modal {
.log-line-default { .log-line-default {
color: var(--fg, #9cdef2); color: var(--fg, #9cdef2);
} }
/* The model-comparison grid hard-codes 2-4 equal columns with no phone
breakpoint that stacks them, so at 390px two models render ~178px columns and
four render ~88px columns. Each column is a full scrolling chat (code blocks,
tool output, vote footer), so the text is unreadably over-wrapped and clipped.
On phones, stack the panes into a single scrollable column instead. */
@media (max-width: 768px) {
.compare-grid[data-cols] { grid-template-columns: 1fr !important; overflow-y: auto; }
.compare-pane { min-height: 60dvh; }
}
@@ -0,0 +1,273 @@
"""Path-confinement regression tests for research routes.
Covers the CodeQL py/path-injection alert cluster (#552-#567) in
routes/research/research_routes.py:
- _owns_in_memory disk fallback (alerts #552, #553)
- _assert_owns_research (alerts #554, #555)
- research_detail (alerts #556, #557)
- research_archive (alerts #558, #559, #560)
- research_delete (alerts #561, #562, #563)
- research_result_peek (alerts #564, #565)
- research_spinoff (alerts #566, #567)
"""
import asyncio
import json
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from fastapi import HTTPException
from routes.research_routes import setup_research_routes
from routes.research.research_routes import _confine_research_path
@pytest.fixture(autouse=True)
def _redirect_research_dir(tmp_path, monkeypatch):
monkeypatch.setattr(
"routes.research_routes.DEEP_RESEARCH_DIR",
str(tmp_path / "deep_research"),
)
def _request(user: str):
return SimpleNamespace(state=SimpleNamespace(current_user=user))
def _route(router, path: str, method: str):
for route in router.routes:
if getattr(route, "path", "") != path:
continue
if method in getattr(route, "methods", set()):
return route.endpoint
raise AssertionError(f"{method} {path} route not registered")
def _write_research(data_dir, session_id: str, **data):
data_dir.mkdir(parents=True, exist_ok=True)
path = data_dir / f"{session_id}.json"
path.write_text(json.dumps(data), encoding="utf-8")
return path
def _research_handler():
handler = MagicMock()
handler._active_tasks = {}
return handler
# ---------------------------------------------------------------------------
# Helper-level tests — _confine_research_path
# ---------------------------------------------------------------------------
def test_confine_allows_valid_session_id(tmp_path, monkeypatch):
data_dir = tmp_path / "deep_research"
data_dir.mkdir()
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
path = _confine_research_path("rp-abc123de4567")
assert path == (data_dir / "rp-abc123de4567.json").resolve()
@pytest.mark.parametrize("bad_id", [
"../escape",
"../../etc/passwd",
"/etc/passwd",
"safe/../../x",
"",
"rp_bad", # underscore not in allowed charset
"rp-bad.json", # dot not in allowed charset
"a" * 129, # exceeds length limit
])
def test_confine_rejects_bad_session_ids(tmp_path, monkeypatch, bad_id):
data_dir = tmp_path / "deep_research"
data_dir.mkdir()
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
with pytest.raises(HTTPException) as exc:
_confine_research_path(bad_id)
assert exc.value.status_code == 400
def test_confine_rejects_symlink_escape(tmp_path, monkeypatch):
"""A symlink inside DEEP_RESEARCH_DIR that resolves outside is rejected."""
data_dir = tmp_path / "deep_research"
outside = tmp_path / "outside"
data_dir.mkdir()
outside.mkdir()
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
target = outside / "rp-linktest1234.json"
target.write_text("{}", encoding="utf-8")
link = data_dir / "rp-linktest1234.json"
try:
link.symlink_to(target)
except (AttributeError, NotImplementedError, OSError) as e:
pytest.skip(f"symlinks unavailable: {e}")
with pytest.raises(HTTPException) as exc:
_confine_research_path("rp-linktest1234")
assert exc.value.status_code == 400
# ---------------------------------------------------------------------------
# Route-level tests — valid paths work
# ---------------------------------------------------------------------------
def test_detail_returns_data_for_owner(tmp_path):
data_dir = tmp_path / "deep_research"
_write_research(data_dir, "rp-validid12345", owner="alice", query="valid query")
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/detail/{session_id}", "GET")
out = asyncio.run(target(session_id="rp-validid12345", request=_request("alice")))
assert out["query"] == "valid query"
# ---------------------------------------------------------------------------
# Route-level tests — traversal and injection rejected
# ---------------------------------------------------------------------------
_TRAVERSAL_IDS = [
"../escape",
"../../etc/passwd",
"/etc/passwd",
"safe/../../x",
"rp_under",
"a" * 129,
]
@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS)
def test_detail_rejects_traversal(bad_id):
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/detail/{session_id}", "GET")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id=bad_id, request=_request("alice")))
assert exc.value.status_code == 400
@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS)
def test_archive_rejects_traversal(bad_id):
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/{session_id}/archive", "POST")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id=bad_id, request=_request("alice"), archived=True))
assert exc.value.status_code == 400
@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS)
def test_delete_rejects_traversal(bad_id):
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/{session_id}", "DELETE")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id=bad_id, request=_request("alice")))
assert exc.value.status_code == 400
# ---------------------------------------------------------------------------
# Route-level tests — traversal does not touch files outside DEEP_RESEARCH_DIR
# ---------------------------------------------------------------------------
def test_delete_traversal_does_not_delete_outside_file(tmp_path, monkeypatch):
data_dir = tmp_path / "deep_research"
data_dir.mkdir(parents=True)
outside = tmp_path / "sensitive.json"
outside.write_text('{"secret": true}', encoding="utf-8")
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/{session_id}", "DELETE")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id="../sensitive", request=_request("alice")))
assert exc.value.status_code == 400
assert outside.exists(), "file outside DEEP_RESEARCH_DIR must not be deleted"
def test_archive_traversal_does_not_mutate_outside_file(tmp_path, monkeypatch):
data_dir = tmp_path / "deep_research"
data_dir.mkdir(parents=True)
outside = tmp_path / "sensitive.json"
outside.write_text('{"owner": "alice", "archived": false}', encoding="utf-8")
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/{session_id}/archive", "POST")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id="../sensitive", request=_request("alice"), archived=True))
assert exc.value.status_code == 400
data = json.loads(outside.read_text(encoding="utf-8"))
assert data["archived"] is False, "file outside DEEP_RESEARCH_DIR must not be mutated"
def test_detail_traversal_does_not_read_outside_file(tmp_path, monkeypatch):
data_dir = tmp_path / "deep_research"
data_dir.mkdir(parents=True)
outside = tmp_path / "sensitive.json"
outside.write_text('{"owner": "alice", "result": "secret data"}', encoding="utf-8")
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/detail/{session_id}", "GET")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id="../sensitive", request=_request("alice")))
assert exc.value.status_code == 400
# ---------------------------------------------------------------------------
# Route-level symlink escape test
# ---------------------------------------------------------------------------
def test_detail_rejects_symlink_escape(tmp_path, monkeypatch):
"""research_detail rejects a confined-format ID whose JSON is a symlink to outside."""
data_dir = tmp_path / "deep_research"
outside_dir = tmp_path / "outside"
data_dir.mkdir(parents=True)
outside_dir.mkdir()
outside_file = outside_dir / "rp-linktest5678.json"
outside_file.write_text(
json.dumps({"owner": "alice", "result": "secret"}), encoding="utf-8"
)
link = data_dir / "rp-linktest5678.json"
try:
link.symlink_to(outside_file)
except (AttributeError, NotImplementedError, OSError) as e:
pytest.skip(f"symlinks unavailable: {e}")
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/detail/{session_id}", "GET")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id="rp-linktest5678", request=_request("alice")))
assert exc.value.status_code == 400
# ---------------------------------------------------------------------------
# Owner/session scoping cannot escape root
# ---------------------------------------------------------------------------
def test_owner_scoped_paths_stay_within_research_root(tmp_path, monkeypatch):
"""Owner-scoped session IDs never produce paths outside DEEP_RESEARCH_DIR."""
data_dir = tmp_path / "deep_research"
data_dir.mkdir(parents=True)
monkeypatch.setattr("routes.research_routes.DEEP_RESEARCH_DIR", str(data_dir))
root = data_dir.resolve()
for session_id in ("rp-abc123456789", "rp-000000000001", "abc-xyz-123"):
path = _confine_research_path(session_id)
assert path.resolve().is_relative_to(root), (
f"{session_id!r} produced path outside research root: {path}"
)
@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS)
def test_result_peek_rejects_traversal(bad_id):
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/result-peek/{session_id}", "POST")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id=bad_id, request=_request("alice")))
assert exc.value.status_code == 400
@pytest.mark.parametrize("bad_id", _TRAVERSAL_IDS)
def test_spinoff_rejects_traversal(bad_id):
router = setup_research_routes(_research_handler())
target = _route(router, "/api/research/spinoff/{session_id}", "POST")
with pytest.raises(HTTPException) as exc:
asyncio.run(target(session_id=bad_id, request=_request("alice")))
assert exc.value.status_code == 400
+295 -6
View File
@@ -894,7 +894,8 @@ def test_web_fetch_guard_fails_closed_on_empty_resolution(monkeypatch):
def test_web_fetch_guard_blocks_redirect_into_private(monkeypatch): def test_web_fetch_guard_blocks_redirect_into_private(monkeypatch):
# A public URL that 302-redirects to an internal address must be blocked # A public URL that 302-redirects to an internal address must be blocked
# at the redirect hop, not followed. # at the redirect hop, not followed. _get_public_url now uses
# httpx.Client(...).stream(...) so the test must mock that path.
import httpx import httpx
from src.search import content from src.search import content
@@ -905,14 +906,31 @@ def test_web_fetch_guard_blocks_redirect_into_private(monkeypatch):
status_code = 302 status_code = 302
url = "http://public.example/start" url = "http://public.example/start"
headers = {"location": "http://169.254.169.254/latest/meta-data/"} headers = {"location": "http://169.254.169.254/latest/meta-data/"}
encoding = "utf-8"
from contextlib import contextmanager class _FakeStream:
def __enter__(self):
return _Resp()
@contextmanager def __exit__(self, *args):
def _fake_stream(method, url, **kwargs): return False
yield _Resp()
monkeypatch.setattr(httpx, "stream", _fake_stream) class _FakeClient:
def __init__(self, *args, **kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *args):
return False
def stream(self, method, url):
assert method == "GET"
assert url == "http://public.example/start"
return _FakeStream()
monkeypatch.setattr(httpx, "Client", _FakeClient)
with _pytest.raises(httpx.RequestError) as exc: with _pytest.raises(httpx.RequestError) as exc:
content._get_public_url("http://public.example/start", headers={}, timeout=5) content._get_public_url("http://public.example/start", headers={}, timeout=5)
@@ -1224,3 +1242,274 @@ def test_visual_report_escapes_request_category():
# value must coerce rather than crash the render (html.escape needs a str). # value must coerce rather than crash the render (html.escape needs a str).
out = generate_visual_report(question="q", report_markdown="## H", category=12345) out = generate_visual_report(question="q", report_markdown="## H", category=12345)
assert "category-12345" in out assert "category-12345" in out
# ── DNS rebinding (audit finding 8.1) ────────────────────────────────
# _resolve_public_ips resolves a URL's hostname once per hop and rejects
# private / metadata targets, but httpx would then re-resolve the
# hostname at connect time. The fix: the actual TCP connect is pinned
# to the resolved IP via a custom httpcore.NetworkBackend, while the
# URL / Host header / SNI stay on the original hostname.
import ipaddress as _ipaddr
import socket as _socket
import threading as _threading
import httpx as _httpx
def test_dns_rebinding_blocked_by_resolve_gate(monkeypatch):
from src.search import content
monkeypatch.setattr(content, "_resolve_hostname_ips",
lambda host: [_ipaddr.ip_address("10.0.0.5")])
with _pytest.raises(_httpx.RequestError) as exc:
content._resolve_public_ips("https://attacker.example/")
assert "non-public" in str(exc.value).lower()
def test_dns_rebinding_pinned_backend_connects_to_resolved_ip(monkeypatch):
"""``_PinnedBackend.connect_tcp`` must ignore the URL's host and
dial the pinned IP at the original port. This is the core of the
fix: httpcore's NetworkBackend contract lets us intercept the
connect before DNS lookup happens.
"""
from src.search import content
pinned_ip = _ipaddr.ip_address("93.184.216.34")
captured = {}
class _StubStream:
def close(self):
pass
class _StubBackend:
def connect_tcp(self, host, port, timeout=None, local_address=None, socket_options=None):
captured["host"] = host
captured["port"] = port
return _StubStream()
def connect_unix_socket(self, path, timeout=None, socket_options=None):
raise OSError("not used")
def sleep(self, seconds):
pass
backend = content._PinnedBackend(pinned_ip)
monkeypatch.setattr(backend, "_real", _StubBackend())
backend.connect_tcp("attacker.example", 443)
assert captured["host"] == "93.184.216.34", captured
assert captured["port"] == 443, captured
def test_dns_rebinding_pinned_transport_dials_pinned_ip(monkeypatch):
"""End-to-end: ``_PinnedTransport`` actually dials the pinned IP
when given a hostname, with the original URL's Host header
preserved. We stand up a local socket server on a free port and
make the transport connect there via the pinned backend.
"""
from src.search import content
import httpcore
# Stand up a TCP server that accepts one connection and records
# the request bytes it received, then returns a minimal HTTP/1.1
# response.
captured = {"request": b""}
server_sock = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
server_sock.bind(("127.0.0.1", 0))
server_sock.listen(1)
port = server_sock.getsockname()[1]
def serve_once():
conn, _ = server_sock.accept()
with conn:
conn.settimeout(2.0)
buf = b""
try:
while b"\r\n\r\n" not in buf:
chunk = conn.recv(4096)
if not chunk:
break
buf += chunk
except _socket.timeout:
pass
captured["request"] = buf
conn.sendall(
b"HTTP/1.1 200 OK\r\n"
b"Content-Length: 2\r\n"
b"Connection: close\r\n"
b"\r\n"
b"OK"
)
t = _threading.Thread(target=serve_once, daemon=True)
t.start()
# Pin the transport to 127.0.0.1:<port>. The caller hands it a URL
# with a fake hostname so we can verify the host header is sent
# while the TCP connect goes to the pinned IP.
pinned_ip = _ipaddr.ip_address("127.0.0.1")
transport = content._PinnedTransport(pinned_ip)
req = _httpx.Request(
"GET",
f"http://attacker.test:{port}/path?q=1",
headers={"host": "attacker.test"},
)
try:
with _httpx.Client(transport=transport, timeout=5) as client:
response = client.send(req)
assert response.status_code == 200, response.text
finally:
server_sock.close()
t.join(timeout=2)
request_bytes = captured["request"]
assert request_bytes, "server never received a request"
# Host header is the original hostname, not the IP. (httpx
# lowercases header names; compare case-insensitively.)
headers_blob = request_bytes.lower()
assert b"host: attacker.test" in headers_blob, request_bytes
# The path was preserved.
assert b"/path?q=1" in request_bytes, request_bytes
def test_dns_rebinding_pinned_transport_preserves_url_netloc(monkeypatch):
"""The URL the transport hands to the underlying httpcore layer
must still be the original ``https://example.com/...`` — never
rewritten to the pinned IP. SNI / vhost depend on this.
"""
from src.search import content
seen_url = {}
class _RecordingPool:
def handle_request(self, req):
seen_url["host"] = req.url.host.decode() if isinstance(req.url.host, bytes) else req.url.host
seen_url["scheme"] = req.url.scheme.decode() if isinstance(req.url.scheme, bytes) else req.url.scheme
seen_url["target"] = req.url.target.decode() if isinstance(req.url.target, bytes) else req.url.target
raise _httpx.ConnectError("intercepted")
def close(self):
pass
pinned_ip = _ipaddr.ip_address("93.184.216.34")
transport = content._PinnedTransport(pinned_ip)
transport._pool = _RecordingPool()
req = _httpx.Request("GET", "https://example.com/some/path?q=1")
with _pytest.raises(_httpx.ConnectError):
transport.handle_request(req)
assert seen_url["host"] == "example.com", seen_url
assert seen_url["scheme"] == "https", seen_url
assert seen_url["target"] == "/some/path?q=1", seen_url
def test_dns_rebinding_redirect_re_resolves_per_hop(monkeypatch):
"""Every redirect hop must call ``_resolve_public_ips`` again.
A redirect to a private-IP target must be blocked even when the
first hop was public.
"""
from src.search import content
seen = []
def fake_resolve(url):
seen.append(url)
if "private" in url:
raise _httpx.RequestError(f"Blocked non-public URL: {url}")
return [_ipaddr.ip_address("93.184.216.34")]
monkeypatch.setattr(content, "_resolve_public_ips", fake_resolve)
class _Resp:
status_code = 302
headers = {"location": "http://private.example/secret"}
encoding = "utf-8"
def __init__(self, url):
self.url = url
class _FakeStream:
def __init__(self, response):
self.response = response
def __enter__(self):
return self.response
def __exit__(self, *args):
return False
class _FakeClient:
def __init__(self, *a, **k):
pass
def __enter__(self):
return self
def __exit__(self, *a):
return False
def stream(self, method, url):
assert method == "GET"
return _FakeStream(_Resp(url))
monkeypatch.setattr(_httpx, "Client", _FakeClient)
with _pytest.raises(_httpx.RequestError) as exc:
content._get_public_url("http://public.example/start", headers={}, timeout=5)
assert "non-public" in str(exc.value).lower()
# Both hops were validated.
assert seen == ["http://public.example/start", "http://private.example/secret"], seen
def test_dns_rebinding_transport_uses_public_apis(monkeypatch):
"""Static guard: ``_PinnedTransport`` must use only the public
``httpx.BaseTransport`` / ``httpcore`` APIs. No subclassing of
``httpx.HTTPTransport`` (whose ``_pool`` slot we'd have to
overwrite), no reads of private ``httpcore.ConnectionPool``
attributes, and no imports from ``httpx._transports``.
"""
from src.search import content
import inspect
# 1) Subclass check: must be BaseTransport, not HTTPTransport.
mro_names = [c.__name__ for c in content._PinnedTransport.__mro__]
assert "BaseTransport" in mro_names, mro_names
assert "HTTPTransport" not in mro_names, (
"_PinnedTransport subclasses httpx.HTTPTransport. Subclass "
"httpx.BaseTransport instead and build the pool from scratch "
"with the public httpcore.ConnectionPool API."
)
# 2) No reads of private httpcore.ConnectionPool attrs.
src = inspect.getsource(content._PinnedTransport)
forbidden = (
"_ssl_context",
"_max_connections",
"_max_keepalive_connections",
"_keepalive_expiry",
"_http1",
"_http2",
"_network_backend",
)
leaked = [name for name in forbidden if name in src]
assert not leaked, (
f"_PinnedTransport reads private httpcore.ConnectionPool attrs: {leaked}. "
"Build the pool from the public httpcore.ConnectionPool API instead."
)
# 3) No imports from httpx's private transport module.
module_src = inspect.getsource(content)
forbidden_imports = ("from httpx._transports", "import httpx._transports")
leaked_imports = [s for s in forbidden_imports if s in module_src]
assert not leaked_imports, (
f"content.py imports from httpx's private transport module: {leaked_imports}. "
"Use only the public httpx and httpcore APIs."
)
+51
View File
@@ -13,6 +13,57 @@ import pytest
from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES
from services.search import content as content_mod from services.search import content as content_mod
import pytest as _pytest_for_client_stream_compat
@_pytest_for_client_stream_compat.fixture(autouse=True)
def _client_stream_compat_for_pinned_fetch(monkeypatch):
"""Adapt old size-cap tests to the current pinned Client.stream path.
These tests monkeypatch httpx.stream(...) to return fake responses. The
production fetcher now uses httpx.Client(...).stream(...) so it can pass a
pinned transport. When a test has replaced httpx.stream, route Client.stream
through that fake. When it has not, fall back to a real Client so unrelated
behavior in this file is not changed.
"""
import httpx
real_client_cls = httpx.Client
original_stream = httpx.stream
class _ClientProxy:
def __init__(self, *args, **kwargs):
self._args = args
self._kwargs = kwargs
self._real_cm = None
self._real_client = None
def __enter__(self):
if httpx.stream is original_stream:
self._real_cm = real_client_cls(*self._args, **self._kwargs)
self._real_client = self._real_cm.__enter__()
return self._real_client
return self
def __exit__(self, *args):
if self._real_cm is not None:
return self._real_cm.__exit__(*args)
return False
def stream(self, method, url):
if self._real_client is not None:
return self._real_client.stream(method, url)
kwargs = {
"headers": self._kwargs.get("headers"),
"timeout": self._kwargs.get("timeout"),
"follow_redirects": self._kwargs.get("follow_redirects"),
}
return httpx.stream(method, url, **kwargs)
monkeypatch.setattr(httpx, "Client", _ClientProxy)
class _FakeStream: class _FakeStream:
"""Stands in for the httpx.stream(...) context manager.""" """Stands in for the httpx.stream(...) context manager."""