mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-14 12:48:03 +00:00
fix(stabilization): harden attachment lifecycle and agent guard signals (#5420)
* fix: harden stabilization attachment and agent guards * fix(uploads): preserve durable references during cleanup * fix(uploads): close cleanup and compaction races
This commit is contained in:
falabellamichael
+41
-2
@@ -3828,9 +3828,8 @@ async def stream_agent_loop(
|
||||
and _intent_match is not None
|
||||
and len(_intent_text) < 400
|
||||
and "```" not in _intent_text
|
||||
and _intent_nudge_count < _MAX_INTENT_NUDGES
|
||||
)
|
||||
if _looks_like_promise:
|
||||
if _looks_like_promise and _intent_nudge_count < _MAX_INTENT_NUDGES:
|
||||
_intent_nudge_count += 1
|
||||
_matched_phrase = _intent_match.group(0).strip()
|
||||
logger.info(f"[agent] intent-without-action nudge #{_intent_nudge_count} on round {round_num}: {_matched_phrase!r}")
|
||||
@@ -3859,6 +3858,31 @@ async def stream_agent_loop(
|
||||
# Visible signal in the stream so the user knows we caught it.
|
||||
yield f'data: {json.dumps({"type": "agent_step", "round": round_num + 1})}\n\n'
|
||||
continue
|
||||
if _looks_like_promise:
|
||||
_matched_phrase = _intent_match.group(0).strip()
|
||||
_guard_message = (
|
||||
"The agent stopped because it repeatedly announced a tool "
|
||||
"action without making the tool call."
|
||||
)
|
||||
logger.warning(
|
||||
"[agent] intent-without-action guard exhausted on round %d after %d nudges: %r",
|
||||
round_num,
|
||||
_intent_nudge_count,
|
||||
_matched_phrase,
|
||||
)
|
||||
yield (
|
||||
"data: "
|
||||
+ json.dumps({
|
||||
"type": "intent_nudge_exhausted",
|
||||
"reason": "intent_without_action_nudge_cap",
|
||||
"message": _guard_message,
|
||||
"round": round_num,
|
||||
"nudges": _intent_nudge_count,
|
||||
"matched": _matched_phrase,
|
||||
})
|
||||
+ "\n\n"
|
||||
)
|
||||
break
|
||||
break # no tools — done
|
||||
|
||||
# ── Loop-breaker (Terminus-style stall detector) ──────────────
|
||||
@@ -3895,6 +3919,21 @@ async def stream_agent_loop(
|
||||
reason = (f"calling {_runaway} with identical arguments over and over" if _runaway
|
||||
else "repeating the same tool calls without new progress")
|
||||
logger.warning(f"[agent] loop-breaker tripped on round {round_num} ({reason}); sig={_sig[:80]!r}")
|
||||
yield (
|
||||
"data: "
|
||||
+ json.dumps({
|
||||
"type": "loop_breaker_triggered",
|
||||
"reason": "loop_breaker_stall",
|
||||
"message": (
|
||||
"The loop-breaker detected repeated tool calls without "
|
||||
"new progress, so the agent is being forced to stop "
|
||||
"using tools and give its best final answer."
|
||||
),
|
||||
"round": round_num,
|
||||
"detail": reason,
|
||||
})
|
||||
+ "\n\n"
|
||||
)
|
||||
# The model has been executing tools, so its results are already
|
||||
# in context. Force ONE tool-free round to converge: write the
|
||||
# answer from what it has, or state plainly what's blocking it.
|
||||
|
||||
@@ -2,10 +2,16 @@ from typing import Any, Dict, List, Optional
|
||||
import logging
|
||||
import re
|
||||
from src.constants import MAX_READ_CHARS
|
||||
from src.tool_utils import _parse_tool_args
|
||||
from src.tool_utils import _parse_tool_args, get_upload_handler
|
||||
from src.upload_handler import reserve_upload_references
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _missing_document_upload(owner: Optional[str], content: Any) -> Optional[str]:
|
||||
"""Reserve explicit upload URLs before an agent persists document text."""
|
||||
return reserve_upload_references(get_upload_handler(), owner, content)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Active document state
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -384,6 +390,13 @@ class CreateDocumentTool:
|
||||
return {"error": "Cannot create document in another user's session"}
|
||||
_owner = _sess.owner if _sess else None
|
||||
|
||||
missing_id = _missing_document_upload(_owner, content)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
doc = Document(
|
||||
id=doc_id,
|
||||
session_id=session_id,
|
||||
@@ -455,6 +468,13 @@ class UpdateDocumentTool:
|
||||
if is_email_doc:
|
||||
doc.language = "email"
|
||||
|
||||
missing_id = _missing_document_upload(owner, new_content)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
if not is_email_doc and _pdf_source_upload_id(doc.current_content or ""):
|
||||
return _create_pdf_text_derivative(
|
||||
db,
|
||||
@@ -532,6 +552,12 @@ class EditDocumentTool:
|
||||
applied = 1
|
||||
skipped = max(0, len(edits) - 1)
|
||||
doc.language = "email"
|
||||
missing_id = _missing_document_upload(owner, updated_content)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
new_ver = doc.version_count + 1
|
||||
ver = DocumentVersion(
|
||||
id=str(uuid.uuid4()),
|
||||
@@ -584,6 +610,13 @@ class EditDocumentTool:
|
||||
if applied == 0:
|
||||
return {"error": f"No edits applied — none of the FIND blocks matched the document content (skipped {skipped})"}
|
||||
|
||||
missing_id = _missing_document_upload(owner, updated_content)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
if _pdf_source_upload_id(doc.current_content or ""):
|
||||
return _create_pdf_text_derivative(
|
||||
db,
|
||||
|
||||
@@ -21,6 +21,7 @@ from src.model_discovery import ModelDiscovery
|
||||
from src.chat_handler import ChatHandler
|
||||
from src.research_handler import ResearchHandler
|
||||
from src.upload_handler import UploadHandler
|
||||
from src.tool_utils import set_upload_handler
|
||||
from src.search import update_search_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -49,6 +50,8 @@ def initialize_managers(base_dir: str, rag_manager=None) -> Dict[str, Any]:
|
||||
session_manager = SessionManager(SESSIONS_FILE)
|
||||
set_session_manager(session_manager) # Enable Session.add_message() persistence
|
||||
upload_handler = UploadHandler(base_dir, UPLOAD_DIR)
|
||||
session_manager.upload_handler = upload_handler
|
||||
set_upload_handler(upload_handler)
|
||||
personal_docs_manager = PersonalDocsManager(PERSONAL_DIR, rag_manager)
|
||||
api_key_manager = APIKeyManager(DATA_DIR)
|
||||
preset_manager = PresetManager(DATA_DIR)
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Attachment reference helpers for chat storage and tool manifests.
|
||||
|
||||
Live model calls may need provider-specific data URLs for the current turn.
|
||||
Persisted history and search indexes should keep stable upload references and
|
||||
human-readable text instead of duplicating raw media bytes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
DATA_URL_RE = re.compile(
|
||||
r"data:[^;,\s\"']+;base64,[A-Za-z0-9+/=]+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
MEDIA_BLOCK_TYPES = {
|
||||
"image",
|
||||
"image_url",
|
||||
"input_image",
|
||||
"audio",
|
||||
"input_audio",
|
||||
"file",
|
||||
}
|
||||
|
||||
|
||||
def strip_inline_data_urls(text: str) -> str:
|
||||
"""Replace inline data URLs with a compact marker."""
|
||||
if not isinstance(text, str) or ";base64," not in text:
|
||||
return text
|
||||
return DATA_URL_RE.sub("[inline media omitted from persisted history]", text)
|
||||
|
||||
|
||||
def attachment_ref(info: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return the stable attachment reference shape used outside raw uploads."""
|
||||
upload_id = str(info.get("id") or info.get("attachment_id") or "").strip()
|
||||
try:
|
||||
size = int(info.get("size") or 0)
|
||||
except (TypeError, ValueError):
|
||||
size = 0
|
||||
ref = {
|
||||
"type": "attachment_ref",
|
||||
"attachment_id": upload_id,
|
||||
"name": info.get("name") or info.get("original_name") or upload_id,
|
||||
"mime": info.get("mime") or "application/octet-stream",
|
||||
"size": size,
|
||||
}
|
||||
checksum = info.get("checksum_sha256") or info.get("sha256") or info.get("hash")
|
||||
if checksum:
|
||||
ref["checksum_sha256"] = checksum
|
||||
created_at = info.get("created_at") or info.get("uploaded_at")
|
||||
if created_at:
|
||||
ref["created_at"] = created_at
|
||||
for key in ("width", "height", "vision", "vision_model", "gallery_id"):
|
||||
value = info.get(key)
|
||||
if value is not None:
|
||||
ref[key] = value
|
||||
return ref
|
||||
|
||||
|
||||
def attachment_refs_from_metadata(metadata: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
"""Extract attachment refs from message metadata."""
|
||||
attachments = (metadata or {}).get("attachments") or []
|
||||
if not isinstance(attachments, list):
|
||||
return []
|
||||
refs: list[dict[str, Any]] = []
|
||||
for item in attachments:
|
||||
if isinstance(item, dict):
|
||||
ref = attachment_ref(item)
|
||||
if ref.get("attachment_id"):
|
||||
refs.append(ref)
|
||||
return refs
|
||||
|
||||
|
||||
def _ref_line(ref: dict[str, Any]) -> str:
|
||||
parts = [f"Attachment: {ref.get('name') or ref.get('attachment_id') or 'upload'}"]
|
||||
if ref.get("attachment_id"):
|
||||
parts.append(f"id={ref['attachment_id']}")
|
||||
if ref.get("mime"):
|
||||
parts.append(f"mime={ref['mime']}")
|
||||
if ref.get("size"):
|
||||
parts.append(f"size={ref['size']} bytes")
|
||||
if ref.get("checksum_sha256"):
|
||||
parts.append(f"sha256={ref['checksum_sha256']}")
|
||||
line = "[" + " | ".join(parts) + "]"
|
||||
if ref.get("vision"):
|
||||
line += f"\n[Attachment description: {str(ref['vision']).strip()}]"
|
||||
return line
|
||||
|
||||
|
||||
def _text_from_blocks(blocks: Iterable[Any]) -> str:
|
||||
lines: list[str] = []
|
||||
omitted_media = 0
|
||||
for block in blocks:
|
||||
if isinstance(block, str):
|
||||
lines.append(strip_inline_data_urls(block))
|
||||
continue
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
block_type = block.get("type")
|
||||
if block_type == "text":
|
||||
text = block.get("text")
|
||||
if isinstance(text, str) and text:
|
||||
lines.append(strip_inline_data_urls(text))
|
||||
elif block_type == "attachment_ref":
|
||||
lines.append(_ref_line(block))
|
||||
elif block_type in MEDIA_BLOCK_TYPES:
|
||||
omitted_media += 1
|
||||
else:
|
||||
try:
|
||||
encoded = json.dumps(block, ensure_ascii=True, sort_keys=True)
|
||||
except TypeError:
|
||||
encoded = str(block)
|
||||
lines.append(strip_inline_data_urls(encoded))
|
||||
if omitted_media:
|
||||
plural = "s" if omitted_media != 1 else ""
|
||||
lines.append(f"[{omitted_media} inline media payload{plural} omitted]")
|
||||
return "\n".join(line for line in lines if line).strip()
|
||||
|
||||
|
||||
def persistable_message_content(
|
||||
content: Any,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""Return content safe for DB persistence and FTS indexing.
|
||||
|
||||
Multimodal provider blocks are collapsed to readable text plus stable
|
||||
attachment reference lines from metadata. This avoids storing base64 media
|
||||
in ``chat_messages.content`` while preserving enough context for reloads,
|
||||
search, and later turns.
|
||||
"""
|
||||
if isinstance(content, list):
|
||||
text = _text_from_blocks(content)
|
||||
refs = attachment_refs_from_metadata(metadata)
|
||||
ref_lines = [_ref_line(ref) for ref in refs]
|
||||
if ref_lines:
|
||||
text = "\n".join([part for part in (text, "\n".join(ref_lines)) if part]).strip()
|
||||
return text
|
||||
if isinstance(content, str):
|
||||
return strip_inline_data_urls(content)
|
||||
try:
|
||||
return strip_inline_data_urls(json.dumps(content, ensure_ascii=True, sort_keys=True))
|
||||
except TypeError:
|
||||
return strip_inline_data_urls(str(content))
|
||||
|
||||
|
||||
def search_index_text(content: Any) -> str:
|
||||
"""Best-effort searchable text for legacy stored content."""
|
||||
if isinstance(content, str):
|
||||
raw = content.strip()
|
||||
if raw.startswith("[") and '"type"' in raw:
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except (TypeError, ValueError):
|
||||
parsed = None
|
||||
if isinstance(parsed, list):
|
||||
return _text_from_blocks(parsed)
|
||||
return strip_inline_data_urls(content)
|
||||
if isinstance(content, list):
|
||||
return _text_from_blocks(content)
|
||||
return persistable_message_content(content)
|
||||
@@ -191,6 +191,8 @@ class ChatHandler:
|
||||
"name": fi.get("name") or fi.get("original_name") or fi["id"],
|
||||
"mime": fi.get("mime", ""),
|
||||
"size": fi.get("size", 0),
|
||||
"checksum_sha256": fi.get("checksum_sha256") or fi.get("hash"),
|
||||
"created_at": fi.get("created_at") or fi.get("uploaded_at"),
|
||||
"width": fi.get("width"),
|
||||
"height": fi.get("height"),
|
||||
})
|
||||
|
||||
@@ -9,6 +9,7 @@ import json
|
||||
from src.constants import MAX_OUTPUT_CHARS
|
||||
|
||||
_mcp_manager = None
|
||||
_upload_handler = None
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCP Manager singleton
|
||||
@@ -23,6 +24,21 @@ def get_mcp_manager():
|
||||
"""Get the global MCP manager instance."""
|
||||
return _mcp_manager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared upload lifecycle handler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def set_upload_handler(handler):
|
||||
"""Register the process's UploadHandler without importing app modules."""
|
||||
global _upload_handler
|
||||
_upload_handler = handler
|
||||
|
||||
|
||||
def get_upload_handler():
|
||||
"""Return the shared UploadHandler used by route and agent writers."""
|
||||
return _upload_handler
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+29
-2
@@ -10,6 +10,8 @@ import re
|
||||
from typing import Dict, Optional
|
||||
|
||||
from src.tools._common import _parse_tool_args
|
||||
from src.tool_utils import get_upload_handler
|
||||
from src.upload_handler import reserve_upload_references
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -408,11 +410,25 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
|
||||
importance = args.get("importance") or "normal"
|
||||
minutes_before = _reminder_minutes(args)
|
||||
|
||||
event_description = _event_description(args, minutes_before)
|
||||
event_location = args.get("location", "") or ""
|
||||
missing_id = reserve_upload_references(
|
||||
get_upload_handler(),
|
||||
owner,
|
||||
event_description,
|
||||
event_location,
|
||||
)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
|
||||
uid = str(_uuid.uuid4())
|
||||
ev = CalendarEvent(
|
||||
uid=uid, calendar_id=cal.id, summary=summary,
|
||||
description=_event_description(args, minutes_before),
|
||||
location=args.get("location", "") or "",
|
||||
description=event_description,
|
||||
location=event_location,
|
||||
dtstart=dtstart, dtend=dtend, all_day=all_day,
|
||||
is_utc=dtstart_is_utc and not all_day,
|
||||
rrule=args.get("rrule", "") or "",
|
||||
@@ -465,6 +481,17 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict:
|
||||
ev = _event_query().filter(CalendarEvent.uid == base_uid).first()
|
||||
if not ev:
|
||||
return {"error": f"Event {uid} not found", "exit_code": 1}
|
||||
missing_id = reserve_upload_references(
|
||||
get_upload_handler(),
|
||||
owner,
|
||||
args.get("description"),
|
||||
args.get("location"),
|
||||
)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
if args.get("summary") is not None:
|
||||
ev.summary = args["summary"]
|
||||
if args.get("description") is not None:
|
||||
|
||||
@@ -10,6 +10,8 @@ import re
|
||||
from typing import Dict, Optional
|
||||
|
||||
from src.tools._common import _parse_tool_args
|
||||
from src.tool_utils import get_upload_handler
|
||||
from src.upload_handler import reserve_upload_references
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -198,6 +200,18 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
|
||||
"duplicate": True,
|
||||
"exit_code": 0,
|
||||
}
|
||||
missing_id = reserve_upload_references(
|
||||
get_upload_handler(),
|
||||
owner,
|
||||
content_raw,
|
||||
args.get("color"),
|
||||
items_json,
|
||||
)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
note = Note(
|
||||
id=str(_uuid.uuid4()),
|
||||
owner=owner,
|
||||
@@ -235,6 +249,19 @@ async def do_manage_notes(content: str, owner: Optional[str] = None) -> Dict:
|
||||
return {"error": f"Note '{note_id}' not found", "exit_code": 1}
|
||||
if not _note_visible_to_owner(note, owner):
|
||||
return {"error": "Note not found", "exit_code": 1}
|
||||
missing_id = reserve_upload_references(
|
||||
get_upload_handler(),
|
||||
owner,
|
||||
args.get("content"),
|
||||
args.get("color"),
|
||||
args.get("checklist_items"),
|
||||
args.get("items"),
|
||||
)
|
||||
if missing_id:
|
||||
return {
|
||||
"error": f"Referenced upload is no longer available: {missing_id}",
|
||||
"exit_code": 1,
|
||||
}
|
||||
for field in ("title", "content", "note_type", "color", "label"):
|
||||
if field in args and args[field] is not None:
|
||||
setattr(note, field, args[field])
|
||||
|
||||
+652
-109
@@ -35,11 +35,34 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UploadCleanupSafetyError(RuntimeError):
|
||||
"""Raised when cleanup cannot prove that destructive work is safe."""
|
||||
|
||||
# The extension is optional: save_upload builds the id as `{uuid.hex}{ext}`,
|
||||
# and a file with no extension (Dockerfile, README, ...) yields a bare 32-hex
|
||||
# id. Requiring `.ext` made those ids fail validation, so the stored file
|
||||
# could never be resolved or downloaded again.
|
||||
UPLOAD_ID_RE = re.compile(r"^[0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?$")
|
||||
UPLOAD_ID_TOKEN_RE = re.compile(
|
||||
r"(?<![0-9a-fA-F])([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)(?![A-Za-z0-9])"
|
||||
)
|
||||
INTERNAL_UPLOAD_URL_RE = re.compile(
|
||||
r"(?:odysseus://attachment/|/api/upload/)"
|
||||
r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
|
||||
r"(?=$|[\s\"'<>\[\](){},;!?:&#]|\.(?![A-Za-z0-9]))"
|
||||
)
|
||||
PDF_SOURCE_UPLOAD_RE = re.compile(
|
||||
r"<!--\s*pdf(?:_form)?_source\b[^>]*\bupload_id="
|
||||
r"[\"']([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)[\"'][^>]*-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ATTACHMENT_REFERENCE_LINE_RE = re.compile(
|
||||
r"\[Attachment:[^\]\r\n]*\|\s*id="
|
||||
r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
|
||||
r"(?:\s*\||\s*\])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def is_valid_upload_id(upload_id: str) -> bool:
|
||||
@@ -47,6 +70,110 @@ def is_valid_upload_id(upload_id: str) -> bool:
|
||||
return UPLOAD_ID_RE.fullmatch(upload_id or "") is not None
|
||||
|
||||
|
||||
def extract_upload_ids(value: Any) -> set[str]:
|
||||
"""Return canonical upload IDs embedded in a persisted URL/text value."""
|
||||
if not isinstance(value, str) or not value:
|
||||
return set()
|
||||
return set(UPLOAD_ID_TOKEN_RE.findall(value))
|
||||
|
||||
|
||||
def extract_internal_upload_ids(value: Any) -> set[str]:
|
||||
"""Return IDs from explicit internal upload references only.
|
||||
|
||||
Cleanup intentionally uses :func:`extract_upload_ids` conservatively, but
|
||||
write-time reservation must not treat an arbitrary 32-hex checksum in note
|
||||
or calendar text as an upload reference. Nested JSON-like values are
|
||||
supported because note checklist items are persisted as structured data.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
found: set[str] = set()
|
||||
for nested in value.values():
|
||||
found.update(extract_internal_upload_ids(nested))
|
||||
return found
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
found: set[str] = set()
|
||||
for nested in value:
|
||||
found.update(extract_internal_upload_ids(nested))
|
||||
return found
|
||||
if not isinstance(value, str) or not value:
|
||||
return set()
|
||||
return (
|
||||
set(INTERNAL_UPLOAD_URL_RE.findall(value))
|
||||
| set(PDF_SOURCE_UPLOAD_RE.findall(value))
|
||||
| set(ATTACHMENT_REFERENCE_LINE_RE.findall(value))
|
||||
)
|
||||
|
||||
|
||||
def reserve_upload_references(
|
||||
upload_handler: Any,
|
||||
owner: Optional[str],
|
||||
*values: Any,
|
||||
) -> Optional[str]:
|
||||
"""Reserve upload IDs in values before a caller persists references.
|
||||
|
||||
Returns the first ID that cannot be owner-checked/reserved, otherwise
|
||||
``None``. A missing handler is treated as no-op for backward-compatible
|
||||
route factories; production wires the shared UploadHandler instance.
|
||||
"""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
upload_ids: set[str] = set()
|
||||
for value in values:
|
||||
upload_ids.update(extract_internal_upload_ids(value))
|
||||
return reserve_upload_ids(upload_handler, owner, upload_ids)
|
||||
|
||||
|
||||
def reserve_upload_ids(
|
||||
upload_handler: Any,
|
||||
owner: Optional[str],
|
||||
upload_ids: Any,
|
||||
) -> Optional[str]:
|
||||
"""Owner-reserve canonical IDs from a trusted structured reference field."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
canonical_ids = {
|
||||
str(upload_id).strip()
|
||||
for upload_id in (upload_ids or [])
|
||||
if is_valid_upload_id(str(upload_id).strip())
|
||||
}
|
||||
for upload_id in sorted(canonical_ids):
|
||||
try:
|
||||
resolved = upload_handler.reserve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
allow_admin=False,
|
||||
)
|
||||
except Exception:
|
||||
resolved = None
|
||||
if not resolved:
|
||||
return upload_id
|
||||
return None
|
||||
|
||||
|
||||
def reserve_message_upload_references(
|
||||
upload_handler: Any,
|
||||
owner: Optional[str],
|
||||
content: Any,
|
||||
metadata: Any = None,
|
||||
) -> Optional[str]:
|
||||
"""Reserve explicit chat references, including structured attachment IDs."""
|
||||
upload_ids = extract_internal_upload_ids(content)
|
||||
if metadata not in (None, ""):
|
||||
if isinstance(metadata, str):
|
||||
metadata = json.loads(metadata)
|
||||
if not isinstance(metadata, dict):
|
||||
raise ValueError("message metadata must be a JSON object")
|
||||
upload_ids.update(extract_internal_upload_ids(metadata))
|
||||
from src.attachment_refs import attachment_refs_from_metadata
|
||||
|
||||
upload_ids.update(
|
||||
str(ref.get("attachment_id") or "").strip()
|
||||
for ref in attachment_refs_from_metadata(metadata)
|
||||
if ref.get("attachment_id")
|
||||
)
|
||||
return reserve_upload_ids(upload_handler, owner, upload_ids)
|
||||
|
||||
|
||||
def _build_upload_id(safe_filename: str) -> str:
|
||||
"""Build a unique upload id whose extension matches UPLOAD_ID_RE.
|
||||
|
||||
@@ -249,43 +376,295 @@ class UploadHandler:
|
||||
|
||||
return True
|
||||
|
||||
def cleanup_old_uploads(self):
|
||||
"""Remove uploaded files older than CLEANUP_DAYS days."""
|
||||
@staticmethod
|
||||
def _parse_upload_timestamp(value: Any) -> Optional[datetime]:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
cutoff_date = datetime.now() - timedelta(days=self.cleanup_days)
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.astimezone().replace(tzinfo=None)
|
||||
return parsed
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _upload_metadata_is_recent(cls, info: Dict[str, Any], cutoff_date: datetime) -> bool:
|
||||
"""Return True when upload metadata records activity inside retention."""
|
||||
for field in ("last_accessed", "created_at", "uploaded_at"):
|
||||
parsed = cls._parse_upload_timestamp(info.get(field))
|
||||
if parsed is None:
|
||||
continue
|
||||
if parsed >= cutoff_date:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _upload_index_keys_for_file(
|
||||
cls,
|
||||
upload_index: Dict[str, Any],
|
||||
upload_id: str,
|
||||
file_path: str,
|
||||
) -> list[str]:
|
||||
"""Find a coherent set of index rows for one physical upload.
|
||||
|
||||
Every related row must agree on ID, canonical path, owner, and a
|
||||
non-empty checksum. Each row must also contain the complete lifecycle
|
||||
timestamps written for new uploads. Ambiguous or incomplete index
|
||||
state cannot authorize destructive cleanup.
|
||||
"""
|
||||
target_path = os.path.normcase(os.path.realpath(file_path))
|
||||
matches: list[str] = []
|
||||
owners: set[str] = set()
|
||||
checksums: set[str] = set()
|
||||
for key, info in upload_index.items():
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
stored_path = info.get("path")
|
||||
stored_real_path = (
|
||||
os.path.normcase(os.path.realpath(stored_path))
|
||||
if isinstance(stored_path, str) and stored_path
|
||||
else None
|
||||
)
|
||||
same_id = info.get("id") == upload_id
|
||||
same_path = stored_real_path == target_path
|
||||
if not same_id and not same_path:
|
||||
continue
|
||||
if not same_id or not same_path:
|
||||
logger.warning(
|
||||
"Skipping ambiguous cleanup candidate %s: related row has id=%r path=%r",
|
||||
file_path,
|
||||
info.get("id"),
|
||||
stored_path,
|
||||
)
|
||||
return []
|
||||
|
||||
owner = info.get("owner")
|
||||
if not isinstance(owner, str) or not owner.strip():
|
||||
logger.warning(
|
||||
"Skipping incomplete cleanup candidate %s: matching row has no owner",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
|
||||
row_checksums = {
|
||||
str(info.get(field)).strip().lower()
|
||||
for field in ("hash", "checksum_sha256")
|
||||
if info.get(field) is not None and str(info.get(field)).strip()
|
||||
}
|
||||
if not row_checksums:
|
||||
logger.warning(
|
||||
"Skipping incomplete cleanup candidate %s: matching row has no checksum",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
if len(row_checksums) != 1:
|
||||
logger.warning(
|
||||
"Skipping ambiguous cleanup candidate %s: matching row has conflicting checksums",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
|
||||
lifecycle_fields = ("uploaded_at", "created_at", "last_accessed")
|
||||
if any(
|
||||
cls._parse_upload_timestamp(info.get(field)) is None
|
||||
for field in lifecycle_fields
|
||||
):
|
||||
logger.warning(
|
||||
"Skipping incomplete cleanup candidate %s: matching row lacks lifecycle timestamps",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
|
||||
matches.append(key)
|
||||
owners.add(owner)
|
||||
checksums.update(row_checksums)
|
||||
|
||||
if len(owners) > 1 or len(checksums) > 1:
|
||||
logger.warning(
|
||||
"Skipping ambiguous cleanup candidate %s: matching rows disagree on owner or checksum",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
return matches
|
||||
|
||||
def cleanup_old_uploads(
|
||||
self,
|
||||
referenced_upload_ids: Optional[set[str]] = None,
|
||||
referenced_upload_hashes: Optional[set[str]] = None,
|
||||
):
|
||||
"""Remove expired uploads proven unreferenced by a complete snapshot.
|
||||
|
||||
``None`` means reference discovery was not completed, so cleanup fails
|
||||
closed and removes nothing. The admin route supplies both sets after
|
||||
scanning persisted chats, documents, and gallery records.
|
||||
"""
|
||||
if referenced_upload_ids is None or referenced_upload_hashes is None:
|
||||
logger.warning("Upload cleanup skipped: persisted reference snapshot unavailable")
|
||||
return 0
|
||||
|
||||
try:
|
||||
cleanup_started_at = datetime.now()
|
||||
cutoff_date = cleanup_started_at - timedelta(days=self.cleanup_days)
|
||||
cleaned_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.upload_dir):
|
||||
if root == self.upload_dir:
|
||||
continue
|
||||
|
||||
path_parts = root.split(os.sep)
|
||||
if len(path_parts) >= 4:
|
||||
|
||||
referenced_ids = {str(value) for value in referenced_upload_ids}
|
||||
referenced_hashes = {str(value) for value in referenced_upload_hashes}
|
||||
uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
|
||||
|
||||
# Keep index mutation and file removal serialized with upload writes.
|
||||
# Each row removal is atomically persisted before the bytes are
|
||||
# deleted; if deletion fails, the previous index is restored.
|
||||
with self._index_lock:
|
||||
current_index = dict(self._load_upload_index(fail_on_error=True))
|
||||
|
||||
for root, dirs, files in os.walk(self.upload_dir, followlinks=False):
|
||||
is_junction = getattr(os.path, "isjunction", lambda _path: False)
|
||||
dirs[:] = [
|
||||
directory
|
||||
for directory in dirs
|
||||
if not os.path.islink(os.path.join(root, directory))
|
||||
and not is_junction(os.path.join(root, directory))
|
||||
]
|
||||
if root == self.upload_dir:
|
||||
continue
|
||||
if not self._inside_upload_dir(root):
|
||||
dirs[:] = []
|
||||
continue
|
||||
|
||||
path_parts = root.split(os.sep)
|
||||
if len(path_parts) < 4:
|
||||
continue
|
||||
try:
|
||||
dir_date = datetime(int(path_parts[-3]), int(path_parts[-2]), int(path_parts[-1]))
|
||||
if dir_date < cutoff_date:
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
try:
|
||||
os.remove(file_path)
|
||||
cleaned_count += 1
|
||||
logger.info(f"Cleaned up old upload: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to remove {file_path}: {e}")
|
||||
|
||||
try:
|
||||
os.rmdir(root)
|
||||
logger.info(f"Removed empty upload directory: {root}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to remove directory {root}: {e}")
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
if dir_date >= cutoff_date:
|
||||
continue
|
||||
|
||||
for file in files:
|
||||
# Reference discovery only understands canonical upload
|
||||
# IDs; unknown files fail closed instead of being swept.
|
||||
if not self.validate_upload_id(file):
|
||||
continue
|
||||
|
||||
file_path = os.path.join(root, file)
|
||||
if not self._inside_upload_dir(file_path):
|
||||
logger.warning(
|
||||
"Skipping cleanup candidate outside upload directory: %s",
|
||||
file_path,
|
||||
)
|
||||
continue
|
||||
matching_keys = self._upload_index_keys_for_file(
|
||||
current_index,
|
||||
file,
|
||||
file_path,
|
||||
)
|
||||
matching_rows = [
|
||||
current_index[key]
|
||||
for key in matching_keys
|
||||
if isinstance(current_index.get(key), dict)
|
||||
]
|
||||
|
||||
# Files without authoritative live index rows are not
|
||||
# eligible for destructive cleanup. Reference hashes,
|
||||
# recency, and ownership cannot be proven for them.
|
||||
if not matching_rows:
|
||||
continue
|
||||
|
||||
is_referenced = file in referenced_ids or any(
|
||||
str(info.get("id") or "") in referenced_ids
|
||||
or str(info.get("hash") or "") in referenced_hashes
|
||||
or str(info.get("checksum_sha256") or "") in referenced_hashes
|
||||
for info in matching_rows
|
||||
)
|
||||
metadata_is_recent = any(
|
||||
self._upload_metadata_is_recent(info, cutoff_date)
|
||||
for info in matching_rows
|
||||
)
|
||||
if is_referenced or metadata_is_recent:
|
||||
continue
|
||||
|
||||
reduced_index = {
|
||||
key: value
|
||||
for key, value in current_index.items()
|
||||
if key not in matching_keys
|
||||
}
|
||||
if matching_keys:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
reduced_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception as e:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
current_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to restore upload indexes after reconciliation failed for %s",
|
||||
file_path,
|
||||
)
|
||||
raise UploadCleanupSafetyError(
|
||||
"upload index rollback failed before file removal"
|
||||
) from e
|
||||
logger.warning(
|
||||
"Failed to reconcile upload index before removing %s: %s",
|
||||
file_path,
|
||||
e,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
# The bytes are already absent. Keep the reduced
|
||||
# lifecycle index instead of recreating a stale row.
|
||||
current_index = reduced_index
|
||||
logger.info(
|
||||
"Reconciled missing expired upload from index: %s",
|
||||
file_path,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
if matching_keys:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
current_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to restore upload index after removal failed for %s",
|
||||
file_path,
|
||||
)
|
||||
raise UploadCleanupSafetyError(
|
||||
"upload index rollback failed after file removal was refused"
|
||||
) from e
|
||||
logger.warning(f"Failed to remove {file_path}: {e}")
|
||||
continue
|
||||
|
||||
current_index = reduced_index
|
||||
cleaned_count += 1
|
||||
logger.info(f"Cleaned up old unreferenced upload: {file_path}")
|
||||
|
||||
try:
|
||||
if not os.listdir(root):
|
||||
os.rmdir(root)
|
||||
logger.info(f"Removed empty upload directory: {root}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to inspect/remove directory {root}: {e}")
|
||||
|
||||
logger.info(f"Upload cleanup completed: {cleaned_count} files removed")
|
||||
return cleaned_count
|
||||
except Exception as e:
|
||||
logger.error(f"Upload cleanup failed: {e}")
|
||||
return 0
|
||||
raise UploadCleanupSafetyError("upload cleanup safety checks failed") from e
|
||||
|
||||
def validate_upload_id(self, upload_id: str) -> bool:
|
||||
"""Validate that the upload ID matches the expected pattern."""
|
||||
@@ -293,71 +672,103 @@ class UploadHandler:
|
||||
|
||||
def _inside_upload_dir(self, path: str) -> bool:
|
||||
"""Check if path is inside the upload directory."""
|
||||
base = os.path.realpath(self.upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
base = os.path.normcase(os.path.realpath(self.upload_dir))
|
||||
p = os.path.normcase(os.path.realpath(path))
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _atomic_write_json(self, path: str, data: dict) -> None:
|
||||
def _atomic_write_json(
|
||||
self,
|
||||
path: str,
|
||||
data: dict,
|
||||
*,
|
||||
sync_backup: bool = False,
|
||||
) -> None:
|
||||
"""Write `data` to `path` atomically: write to a temp file in the
|
||||
same directory, then `os.replace` onto the target. The kernel
|
||||
guarantees `os.replace` is atomic on POSIX, so a reader either
|
||||
sees the old contents or the new contents, never a half-written
|
||||
file. Also keeps a `.bak` sibling of the previous good state.
|
||||
file. Normally `.bak` retains the previous good state. Destructive
|
||||
lifecycle transitions use ``sync_backup=True`` so recovery cannot
|
||||
resurrect metadata for bytes that were deliberately removed.
|
||||
"""
|
||||
directory = os.path.dirname(path) or "."
|
||||
fd, tmp = tempfile.mkstemp(prefix=".uploads-", suffix=".tmp", dir=directory)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
if os.path.exists(path):
|
||||
bak = path + ".bak"
|
||||
|
||||
def _replace_json(target: str) -> None:
|
||||
fd, tmp = tempfile.mkstemp(
|
||||
prefix=".uploads-",
|
||||
suffix=".tmp",
|
||||
dir=directory,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, target)
|
||||
except Exception:
|
||||
try:
|
||||
shutil.copy2(path, bak)
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, path)
|
||||
# Update cache if this is the main index
|
||||
if path.endswith("uploads.json"):
|
||||
self._index_cache = data
|
||||
try:
|
||||
self._index_mtime = os.path.getmtime(path)
|
||||
except OSError:
|
||||
self._index_mtime = time.time()
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
if sync_backup:
|
||||
_replace_json(path + ".bak")
|
||||
elif os.path.exists(path):
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
shutil.copy2(path, path + ".bak")
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _load_upload_index(self) -> Dict[str, Any]:
|
||||
_replace_json(path)
|
||||
# Update cache if this is the main index
|
||||
if path.endswith("uploads.json"):
|
||||
self._index_cache = data
|
||||
try:
|
||||
self._index_mtime = os.path.getmtime(path)
|
||||
except OSError:
|
||||
self._index_mtime = time.time()
|
||||
|
||||
def _load_upload_index(self, *, fail_on_error: bool = False) -> Dict[str, Any]:
|
||||
"""Load the upload index from disk/cache. Uses mtime-based validation
|
||||
to avoid redundant parsing on hot paths.
|
||||
to avoid redundant parsing on hot paths. When ``fail_on_error`` is
|
||||
true, a missing, malformed, or unreadable live index raises so
|
||||
destructive callers cannot mistake corruption for an empty store.
|
||||
"""
|
||||
uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
|
||||
if not os.path.exists(uploads_db_path):
|
||||
candidates = (uploads_db_path, uploads_db_path + ".bak")
|
||||
if fail_on_error:
|
||||
# A backup is intentionally the previous snapshot. It is useful for
|
||||
# non-destructive reads, but cannot authorize deletion when the live
|
||||
# index is missing or corrupt.
|
||||
if not os.path.exists(uploads_db_path):
|
||||
raise ValueError("live uploads database is missing")
|
||||
existing_candidates = [uploads_db_path]
|
||||
else:
|
||||
existing_candidates = [path for path in candidates if os.path.exists(path)]
|
||||
if not existing_candidates:
|
||||
self._index_cache = {}
|
||||
self._index_mtime = 0.0
|
||||
return {}
|
||||
|
||||
# Check cache validity
|
||||
try:
|
||||
mtime = os.path.getmtime(uploads_db_path)
|
||||
if self._index_cache is not None and mtime <= self._index_mtime:
|
||||
mtime = max(os.path.getmtime(path) for path in existing_candidates)
|
||||
if (
|
||||
not fail_on_error
|
||||
and self._index_cache is not None
|
||||
and mtime <= self._index_mtime
|
||||
):
|
||||
return self._index_cache
|
||||
except OSError:
|
||||
mtime = 0.0
|
||||
|
||||
# Try the live file first, fall back to the .bak sibling if the
|
||||
# live file is truncated/corrupted.
|
||||
for candidate in (uploads_db_path, uploads_db_path + ".bak"):
|
||||
if not os.path.exists(candidate):
|
||||
continue
|
||||
for candidate in existing_candidates:
|
||||
try:
|
||||
with open(candidate, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
@@ -369,6 +780,8 @@ class UploadHandler:
|
||||
logger.warning(f"Failed to read uploads database ({candidate}): {e}")
|
||||
continue
|
||||
|
||||
if fail_on_error:
|
||||
raise ValueError("live uploads database is unreadable")
|
||||
self._index_cache = {}
|
||||
return {}
|
||||
|
||||
@@ -381,6 +794,144 @@ class UploadHandler:
|
||||
return dict(info)
|
||||
return None
|
||||
|
||||
def reserve_upload(
|
||||
self,
|
||||
upload_id: str,
|
||||
*,
|
||||
owner: Optional[str],
|
||||
auth_manager: Any = None,
|
||||
allow_admin: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Owner-check and reserve an indexed upload against cleanup.
|
||||
|
||||
The live index lookup, ownership/path validation, and access touch all
|
||||
occur under the cleanup lock. A durable-reference writer must not
|
||||
commit when this returns ``None``.
|
||||
"""
|
||||
if not self.validate_upload_id(upload_id):
|
||||
return None
|
||||
|
||||
auth_configured = bool(auth_manager and getattr(auth_manager, "is_configured", False))
|
||||
if auth_configured and not owner:
|
||||
return None
|
||||
|
||||
uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
|
||||
with self._index_lock:
|
||||
try:
|
||||
current = dict(self._load_upload_index(fail_on_error=True))
|
||||
except Exception:
|
||||
logger.warning("Cannot reserve upload %s without a valid live index", upload_id)
|
||||
return None
|
||||
matching_keys = [
|
||||
key
|
||||
for key, info in current.items()
|
||||
if isinstance(info, dict) and info.get("id") == upload_id
|
||||
]
|
||||
if not matching_keys:
|
||||
return None
|
||||
|
||||
matching_rows = [dict(current[key]) for key in matching_keys]
|
||||
row_owners = {
|
||||
str(row.get("owner")) if row.get("owner") is not None else None
|
||||
for row in matching_rows
|
||||
}
|
||||
row_hashes = {
|
||||
str(row.get("hash") or row.get("checksum_sha256"))
|
||||
for row in matching_rows
|
||||
if row.get("hash") or row.get("checksum_sha256")
|
||||
}
|
||||
if len(row_owners) != 1 or len(row_hashes) > 1:
|
||||
logger.warning(
|
||||
"Cannot reserve ambiguous upload index rows for %s",
|
||||
upload_id,
|
||||
)
|
||||
return None
|
||||
|
||||
is_admin = False
|
||||
if allow_admin and owner and auth_manager and hasattr(auth_manager, "is_admin"):
|
||||
try:
|
||||
is_admin = bool(auth_manager.is_admin(owner))
|
||||
except Exception:
|
||||
is_admin = False
|
||||
|
||||
now = datetime.now()
|
||||
current_info = matching_rows[0]
|
||||
if owner and not is_admin and current_info.get("owner") != owner:
|
||||
return None
|
||||
if not owner and current_info.get("owner") is not None:
|
||||
return None
|
||||
|
||||
existing_paths: set[str] = set()
|
||||
for row in matching_rows:
|
||||
stored_path = row.get("path")
|
||||
if not stored_path:
|
||||
continue
|
||||
if not self._inside_upload_dir(stored_path):
|
||||
logger.warning(
|
||||
"Cannot reserve upload %s with an out-of-root index path",
|
||||
upload_id,
|
||||
)
|
||||
return None
|
||||
if os.path.isfile(stored_path):
|
||||
if os.path.basename(stored_path) != upload_id:
|
||||
return None
|
||||
existing_paths.add(os.path.normcase(os.path.realpath(stored_path)))
|
||||
if len(existing_paths) > 1:
|
||||
logger.warning("Cannot reserve upload %s with multiple indexed paths", upload_id)
|
||||
return None
|
||||
path = next(iter(existing_paths), None) or self._find_upload_path(upload_id)
|
||||
if not path or not os.path.isfile(path) or not self._inside_upload_dir(path):
|
||||
return None
|
||||
|
||||
last_accessed = self._parse_upload_timestamp(current_info.get("last_accessed"))
|
||||
path_changed = current_info.get("path") != path
|
||||
needs_write = (
|
||||
path_changed
|
||||
or last_accessed is None
|
||||
or last_accessed < now - timedelta(minutes=5)
|
||||
)
|
||||
if needs_write:
|
||||
accessed_at = now.isoformat()
|
||||
updated_index = dict(current)
|
||||
for key in matching_keys:
|
||||
updated = dict(updated_index[key])
|
||||
updated["path"] = path
|
||||
updated["last_accessed"] = accessed_at
|
||||
updated_index[key] = updated
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
updated_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
current,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to restore upload indexes after reservation failed for %s",
|
||||
upload_id,
|
||||
)
|
||||
logger.exception("Failed to reserve upload %s against cleanup", upload_id)
|
||||
return None
|
||||
current_info = dict(updated_index[matching_keys[0]])
|
||||
|
||||
resolved = dict(current_info)
|
||||
resolved.setdefault("id", upload_id)
|
||||
resolved["path"] = path
|
||||
resolved.setdefault("name", os.path.basename(path))
|
||||
resolved.setdefault("original_name", resolved["name"])
|
||||
resolved.setdefault("mime", mimetypes.guess_type(path)[0] or "application/octet-stream")
|
||||
if resolved.get("hash") and not resolved.get("checksum_sha256"):
|
||||
resolved["checksum_sha256"] = resolved["hash"]
|
||||
if resolved.get("uploaded_at") and not resolved.get("created_at"):
|
||||
resolved["created_at"] = resolved["uploaded_at"]
|
||||
return resolved
|
||||
|
||||
def _renamed_upload_index_key(self, key: str, info: Dict[str, Any], old_owner: str, new_owner: str) -> str:
|
||||
"""Return the storage key to use after renaming an owned upload row.
|
||||
|
||||
@@ -475,16 +1026,32 @@ class UploadHandler:
|
||||
if not self.validate_upload_id(upload_id):
|
||||
return None
|
||||
|
||||
candidates: list[str] = []
|
||||
direct = os.path.join(self.upload_dir, upload_id)
|
||||
if os.path.exists(direct) and self._inside_upload_dir(direct):
|
||||
return direct
|
||||
if os.path.isfile(direct) and self._inside_upload_dir(direct):
|
||||
candidates.append(os.path.realpath(direct))
|
||||
|
||||
for root, _dirs, files in os.walk(self.upload_dir, followlinks=False):
|
||||
for root, dirs, files in os.walk(self.upload_dir, followlinks=False):
|
||||
is_junction = getattr(os.path, "isjunction", lambda _path: False)
|
||||
dirs[:] = [
|
||||
directory
|
||||
for directory in dirs
|
||||
if not os.path.islink(os.path.join(root, directory))
|
||||
and not is_junction(os.path.join(root, directory))
|
||||
]
|
||||
if upload_id in files:
|
||||
path = os.path.join(root, upload_id)
|
||||
if self._inside_upload_dir(path):
|
||||
return path
|
||||
return None
|
||||
if os.path.isfile(path) and self._inside_upload_dir(path):
|
||||
real_path = os.path.realpath(path)
|
||||
if real_path not in candidates:
|
||||
candidates.append(real_path)
|
||||
if len(candidates) > 1:
|
||||
logger.warning(
|
||||
"Upload ID %s resolves to multiple physical files",
|
||||
upload_id,
|
||||
)
|
||||
return None
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
def resolve_upload(
|
||||
self,
|
||||
@@ -493,52 +1060,20 @@ class UploadHandler:
|
||||
auth_manager: Any = None,
|
||||
allow_admin: bool = True,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Resolve an upload ID to metadata only if the caller may read it.
|
||||
"""Resolve and reserve an upload only if the caller may read it.
|
||||
|
||||
This is the owner-aware lookup used by internal processors. Public
|
||||
download routes already perform owner checks; chat/document paths must
|
||||
do the same before reading file bytes server-side.
|
||||
do the same before reading file bytes server-side. Reservation shares
|
||||
cleanup's lifecycle lock and prevents a newly persisted reference from
|
||||
racing final deletion.
|
||||
"""
|
||||
if not self.validate_upload_id(upload_id):
|
||||
logger.warning(f"Invalid upload ID format: {upload_id}")
|
||||
return None
|
||||
|
||||
auth_configured = bool(auth_manager and getattr(auth_manager, "is_configured", False))
|
||||
if auth_configured and not owner:
|
||||
return None
|
||||
|
||||
info = self.get_upload_info(upload_id) or {}
|
||||
is_admin = False
|
||||
if allow_admin and owner and auth_manager and hasattr(auth_manager, "is_admin"):
|
||||
try:
|
||||
is_admin = bool(auth_manager.is_admin(owner))
|
||||
except Exception:
|
||||
is_admin = False
|
||||
|
||||
if owner and not is_admin:
|
||||
if info.get("owner") != owner:
|
||||
logger.warning("Upload %s denied for owner %s", upload_id, owner)
|
||||
return None
|
||||
if not owner and info.get("owner") is not None:
|
||||
logger.warning("Upload %s denied without an authenticated owner", upload_id)
|
||||
return None
|
||||
|
||||
path = info.get("path")
|
||||
if not path or not os.path.exists(path) or not self._inside_upload_dir(path):
|
||||
path = self._find_upload_path(upload_id)
|
||||
if not path:
|
||||
return None
|
||||
if not self._inside_upload_dir(path):
|
||||
logger.warning(f"Upload path outside upload directory: {path}")
|
||||
return None
|
||||
|
||||
resolved = dict(info)
|
||||
resolved.setdefault("id", upload_id)
|
||||
resolved["path"] = path
|
||||
resolved.setdefault("name", os.path.basename(path))
|
||||
resolved.setdefault("original_name", resolved["name"])
|
||||
resolved.setdefault("mime", mimetypes.guess_type(path)[0] or "application/octet-stream")
|
||||
return resolved
|
||||
return self.reserve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
allow_admin=allow_admin,
|
||||
)
|
||||
|
||||
def cleanup_rate_limits(self):
|
||||
"""Remove stale entries from upload_rate_log."""
|
||||
@@ -706,6 +1241,9 @@ class UploadHandler:
|
||||
# fresh-insert path below; release the lock first.
|
||||
raise LookupError("upload entry vanished mid-dedupe")
|
||||
existing_file["last_accessed"] = datetime.now().isoformat()
|
||||
existing_file.setdefault("checksum_sha256", file_hash)
|
||||
if existing_file.get("uploaded_at"):
|
||||
existing_file.setdefault("created_at", existing_file["uploaded_at"])
|
||||
current[live_key] = existing_file
|
||||
self._atomic_write_json(uploads_db_path, current)
|
||||
except LookupError:
|
||||
@@ -721,7 +1259,9 @@ class UploadHandler:
|
||||
"size": existing_file["size"],
|
||||
"name": existing_file["original_name"],
|
||||
"hash": file_hash,
|
||||
"checksum_sha256": existing_file.get("checksum_sha256") or file_hash,
|
||||
"uploaded_at": existing_file["uploaded_at"],
|
||||
"created_at": existing_file.get("created_at") or existing_file["uploaded_at"],
|
||||
"owner": existing_file.get("owner"),
|
||||
"width": existing_file.get("width"),
|
||||
"height": existing_file.get("height"),
|
||||
@@ -744,6 +1284,7 @@ class UploadHandler:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
|
||||
# Create file metadata
|
||||
created_at = datetime.now().isoformat()
|
||||
file_metadata = {
|
||||
"id": file_id,
|
||||
"path": file_path,
|
||||
@@ -751,9 +1292,11 @@ class UploadHandler:
|
||||
"size": file_size,
|
||||
"name": safe_filename,
|
||||
"hash": file_hash,
|
||||
"checksum_sha256": file_hash,
|
||||
"original_name": original_filename,
|
||||
"uploaded_at": datetime.now().isoformat(),
|
||||
"last_accessed": datetime.now().isoformat(),
|
||||
"uploaded_at": created_at,
|
||||
"created_at": created_at,
|
||||
"last_accessed": created_at,
|
||||
"client_ip": client_ip,
|
||||
"owner": owner,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user