mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-13 12:38:02 +00:00
Merge remote-tracking branch 'origin/dev'
# Conflicts: # routes/contacts_routes.py
This commit is contained in:
pewdiepie-archdaemon
+206
-62
@@ -8,11 +8,13 @@ import os
|
||||
import re
|
||||
import logging
|
||||
import socket
|
||||
import ssl
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List
|
||||
from typing import Iterable, List, cast
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
import httpcore
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.constants import WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT
|
||||
@@ -91,6 +93,148 @@ def _public_http_url(url: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_public_ips(url: str) -> list[ipaddress._BaseAddress]:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
raise httpx.RequestError(f"Blocked non-public URL: {url}")
|
||||
host = (parsed.hostname or "").strip().lower()
|
||||
if host in ("localhost", "metadata", "metadata.google.internal"):
|
||||
raise httpx.RequestError(f"Blocked non-public hostname: {host}")
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
if _is_private_address(ip):
|
||||
raise httpx.RequestError(f"Blocked non-public IP literal: {host}")
|
||||
return [ip]
|
||||
except httpx.RequestError:
|
||||
raise
|
||||
except ValueError:
|
||||
pass
|
||||
addrs = _resolve_hostname_ips(host)
|
||||
if not addrs or any(_is_private_address(a) for a in addrs):
|
||||
raise httpx.RequestError(f"Blocked non-public URL: {url}")
|
||||
return addrs
|
||||
|
||||
|
||||
class _PinnedBackend(httpcore.NetworkBackend):
|
||||
"""Network backend that connects to a pre-resolved IP.
|
||||
|
||||
httpcore derives the TLS SNI and the ``Host`` header from the URL's
|
||||
origin, not from the host argument passed to ``connect_tcp``. So
|
||||
routing the TCP connect to a resolved IP while leaving the URL
|
||||
untouched keeps SNI / vhost behaviour correct and closes the
|
||||
DNS-rebinding TOCTOU between the SSRF check and the connect.
|
||||
"""
|
||||
|
||||
def __init__(self, ip: ipaddress._BaseAddress):
|
||||
self._ip = str(ip)
|
||||
self._real = httpcore.SyncBackend()
|
||||
|
||||
def connect_tcp(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
timeout: float | None = None,
|
||||
local_address: str | None = None,
|
||||
socket_options=None,
|
||||
):
|
||||
return self._real.connect_tcp(
|
||||
self._ip, port, timeout, local_address, socket_options
|
||||
)
|
||||
|
||||
def connect_unix_socket(self, path, timeout=None, socket_options=None):
|
||||
return self._real.connect_unix_socket(path, timeout, socket_options)
|
||||
|
||||
def sleep(self, seconds: float) -> None:
|
||||
return self._real.sleep(seconds)
|
||||
|
||||
|
||||
# Map httpcore exception classes to their httpx equivalents. Built
|
||||
# once at import time from the public exception classes; avoids any
|
||||
# import of httpx's private transport machinery. httpcore's
|
||||
# ``ConnectionNotAvailable`` is a pool-internal signal (the pool will
|
||||
# close and retry on its own) — we never expect to see it surface to
|
||||
# a transport caller, so it has no httpx counterpart here.
|
||||
_HTTPCORE_TO_HTTPX_EXC = {
|
||||
httpcore.ConnectError: httpx.ConnectError,
|
||||
httpcore.ConnectTimeout: httpx.ConnectTimeout,
|
||||
httpcore.LocalProtocolError: httpx.LocalProtocolError,
|
||||
httpcore.NetworkError: httpx.NetworkError,
|
||||
httpcore.PoolTimeout: httpx.PoolTimeout,
|
||||
httpcore.ProtocolError: httpx.ProtocolError,
|
||||
httpcore.ProxyError: httpx.ProxyError,
|
||||
httpcore.ReadError: httpx.ReadError,
|
||||
httpcore.ReadTimeout: httpx.ReadTimeout,
|
||||
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
|
||||
httpcore.TimeoutException: httpx.TimeoutException,
|
||||
httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
|
||||
httpcore.WriteError: httpx.WriteError,
|
||||
httpcore.WriteTimeout: httpx.WriteTimeout,
|
||||
}
|
||||
|
||||
|
||||
class _PinnedTransport(httpx.BaseTransport):
|
||||
"""Transport that pins every TCP connect to a pre-resolved IP.
|
||||
|
||||
Uses only the public ``httpcore`` and ``httpx`` APIs — no
|
||||
subclassing of ``httpx.HTTPTransport``, no reads of private
|
||||
``httpcore.ConnectionPool`` attributes, no imports from
|
||||
``httpx private transport internals``. The URL is passed through unchanged so SNI
|
||||
/ vhost work as if httpx had been given the hostname directly;
|
||||
only the TCP destination is pinned, closing the DNS-rebinding
|
||||
TOCTOU between the SSRF check and the connect.
|
||||
"""
|
||||
|
||||
def __init__(self, ip: ipaddress._BaseAddress, *, http2: bool = False):
|
||||
self._pool = httpcore.ConnectionPool(
|
||||
ssl_context=ssl.create_default_context(),
|
||||
http1=True,
|
||||
http2=http2,
|
||||
network_backend=_PinnedBackend(ip),
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
self._pool.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
|
||||
self._pool.__exit__(exc_type, exc_value, traceback)
|
||||
|
||||
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
||||
httpcore_req = httpcore.Request(
|
||||
method=request.method,
|
||||
url=httpcore.URL(
|
||||
scheme=request.url.raw_scheme,
|
||||
host=request.url.raw_host,
|
||||
port=request.url.port,
|
||||
target=request.url.raw_path,
|
||||
),
|
||||
headers=request.headers.raw,
|
||||
content=request.stream,
|
||||
extensions=request.extensions,
|
||||
)
|
||||
try:
|
||||
httpcore_resp = self._pool.handle_request(httpcore_req)
|
||||
# Eager materialisation matches the original
|
||||
# ``response.text`` usage in fetch_webpage_content. The
|
||||
# sync pool's stream is a plain Iterable[bytes] despite
|
||||
# the httpcore type hint unioning the async variant.
|
||||
content = b"".join(cast(Iterable[bytes], httpcore_resp.stream))
|
||||
except Exception as exc:
|
||||
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
|
||||
if mapped is not None:
|
||||
raise mapped(str(exc)) from exc
|
||||
raise
|
||||
|
||||
return httpx.Response(
|
||||
status_code=httpcore_resp.status,
|
||||
headers=httpcore_resp.headers,
|
||||
content=content,
|
||||
extensions=httpcore_resp.extensions,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self._pool.close()
|
||||
|
||||
class BodyTooLargeError(Exception):
|
||||
"""The server declared a body larger than the hard fetch ceiling."""
|
||||
|
||||
@@ -141,78 +285,78 @@ class _CappedFetch:
|
||||
|
||||
def _get_public_url(url: str, headers: dict, timeout: int, max_redirects: int = 5,
|
||||
max_bytes: int = None) -> "_CappedFetch":
|
||||
"""Capped streaming GET with SSRF-guarded manual redirects.
|
||||
"""Capped streaming GET with SSRF-guarded, DNS-pinned manual redirects.
|
||||
|
||||
The body is streamed and buffering stops at ``max_bytes`` (default: the
|
||||
soft cap), so an oversized resource cannot be pulled into memory or the
|
||||
content cache in full. When Content-Length already declares a body over
|
||||
the hard ceiling, the fetch is refused before any body bytes are read.
|
||||
Each hop is resolved once, validated as public, and then the actual TCP
|
||||
connection is pinned to that resolved IP. The request URL is left unchanged
|
||||
so Host and TLS SNI keep the original hostname.
|
||||
"""
|
||||
cap = min(max_bytes or WEB_FETCH_SOFT_MAX_BYTES, WEB_FETCH_HARD_MAX_BYTES)
|
||||
current = url
|
||||
for _ in range(max_redirects + 1):
|
||||
if not _public_http_url(current):
|
||||
raise httpx.RequestError("Blocked private/internal URL", request=httpx.Request("GET", current))
|
||||
ips = _resolve_public_ips(current)
|
||||
|
||||
# Force identity transfer-encoding. With gzip/deflate the wire bytes
|
||||
# (and Content-Length) can be a small fraction of the decoded body, so
|
||||
# a tiny compressed response could pass the hard-cap preflight and then
|
||||
# expand past the ceiling in a single decoded chunk before the streamed
|
||||
# cap below can slice it. Identity makes Content-Length the true body
|
||||
# size and keeps each streamed chunk bounded by the network read.
|
||||
# and Content-Length can be a small fraction of the decoded body, so a
|
||||
# tiny compressed response could pass the hard-cap preflight and then
|
||||
# expand past the ceiling in one decoded chunk before the streamed cap
|
||||
# below can slice it.
|
||||
req_headers = dict(headers or {})
|
||||
req_headers["Accept-Encoding"] = "identity"
|
||||
with httpx.stream("GET", current, headers=req_headers, timeout=timeout,
|
||||
follow_redirects=False) as response:
|
||||
if response.status_code in (301, 302, 303, 307, 308):
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
return _CappedFetch(response.status_code, response.headers, b"",
|
||||
False, None, response.encoding, str(response.url))
|
||||
current = urljoin(str(response.url), location)
|
||||
continue
|
||||
|
||||
# A server can ignore the identity request and still return a
|
||||
# compressed body; httpx.iter_bytes would then decode it, and a tiny
|
||||
# gzip can balloon into one decoded chunk far past the cap before we
|
||||
# slice. Refuse a compressed Content-Encoding so the streamed cap
|
||||
# stays a real memory bound (Content-Length is the compressed wire
|
||||
# length here, so the preflight and size metadata are unreliable too).
|
||||
enc = (response.headers.get("content-encoding") or "").strip().lower()
|
||||
if enc and enc != "identity":
|
||||
raise httpx.RequestError(
|
||||
f"Refusing compressed response (Content-Encoding: {enc}) after "
|
||||
"requesting identity: cannot bound decoded body size",
|
||||
request=httpx.Request("GET", current),
|
||||
)
|
||||
with httpx.Client(
|
||||
headers=req_headers,
|
||||
timeout=timeout,
|
||||
follow_redirects=False,
|
||||
transport=_PinnedTransport(ips[0]),
|
||||
) as client:
|
||||
with client.stream("GET", current) as response:
|
||||
if response.status_code in (301, 302, 303, 307, 308):
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
return _CappedFetch(response.status_code, response.headers, b"",
|
||||
False, None, response.encoding, str(response.url))
|
||||
current = urljoin(str(response.url), location)
|
||||
continue
|
||||
|
||||
declared = None
|
||||
raw_len = response.headers.get("content-length")
|
||||
if raw_len and raw_len.isdigit():
|
||||
declared = int(raw_len)
|
||||
# Refuse before buffering anything when the server already tells
|
||||
# us the body exceeds the absolute ceiling (Content-Length is wire
|
||||
# bytes; the decompressed body can only be larger).
|
||||
if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
|
||||
raise BodyTooLargeError(current, declared)
|
||||
# A server can ignore the identity request and still return a
|
||||
# compressed body; httpx.iter_bytes would then decode it, and a
|
||||
# tiny gzip can balloon into one decoded chunk far past the cap.
|
||||
# Refuse compressed Content-Encoding so the streamed cap stays
|
||||
# a real memory bound.
|
||||
enc = (response.headers.get("content-encoding") or "").strip().lower()
|
||||
if enc and enc != "identity":
|
||||
raise httpx.RequestError(
|
||||
f"Refusing compressed response (Content-Encoding: {enc}) after "
|
||||
"requesting identity: cannot bound decoded body size",
|
||||
request=httpx.Request("GET", current),
|
||||
)
|
||||
|
||||
declared = None
|
||||
raw_len = response.headers.get("content-length")
|
||||
if raw_len and raw_len.isdigit():
|
||||
declared = int(raw_len)
|
||||
|
||||
if declared is not None and declared > WEB_FETCH_HARD_MAX_BYTES:
|
||||
raise BodyTooLargeError(current, declared)
|
||||
|
||||
chunks = []
|
||||
read = 0
|
||||
truncated = False
|
||||
for chunk in response.iter_bytes():
|
||||
read += len(chunk)
|
||||
if read > cap:
|
||||
keep = cap - (read - len(chunk))
|
||||
if keep > 0:
|
||||
chunks.append(chunk[:keep])
|
||||
truncated = True
|
||||
break
|
||||
chunks.append(chunk)
|
||||
|
||||
return _CappedFetch(response.status_code, response.headers,
|
||||
b"".join(chunks), truncated, declared,
|
||||
response.encoding, str(response.url))
|
||||
|
||||
chunks = []
|
||||
read = 0
|
||||
truncated = False
|
||||
# We requested identity above, so iter_bytes yields the raw body in
|
||||
# network-read-sized chunks (no decompression expansion); the cap
|
||||
# therefore bounds what we actually buffer.
|
||||
for chunk in response.iter_bytes():
|
||||
read += len(chunk)
|
||||
if read > cap:
|
||||
keep = cap - (read - len(chunk))
|
||||
if keep > 0:
|
||||
chunks.append(chunk[:keep])
|
||||
truncated = True
|
||||
break
|
||||
chunks.append(chunk)
|
||||
return _CappedFetch(response.status_code, response.headers,
|
||||
b"".join(chunks), truncated, declared,
|
||||
response.encoding, str(response.url))
|
||||
raise httpx.RequestError("Too many redirects", request=httpx.Request("GET", current))
|
||||
|
||||
# PDF extraction (optional dependency)
|
||||
|
||||
@@ -34,8 +34,14 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
|
||||
cleaned = query
|
||||
if qtype:
|
||||
cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
|
||||
for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
|
||||
entities["names"].append(token)
|
||||
# Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
|
||||
# class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
|
||||
# "São" (shredded). Keep the ASCII behaviour — the word boundary already
|
||||
# excludes camelCase mid-word capitals — by requiring an all-alphabetic
|
||||
# token of length > 1 whose first character is uppercase.
|
||||
for token in re.findall(r"\b\w+\b", cleaned):
|
||||
if len(token) > 1 and token[0].isupper() and token.isalpha():
|
||||
entities["names"].append(token)
|
||||
for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
|
||||
entities["dates"].append(year)
|
||||
month_day_year = re.findall(
|
||||
|
||||
Reference in New Issue
Block a user