fix(agent): preserve loop guard stream behavior

src/agent_loop.py

@@ -11,7 +11,6 @@ import collections
 import json
 import re
 import time
-import re
 import logging
 from typing import AsyncGenerator, List, Dict, Optional, Set
 from urllib.parse import urlparse
@@ -39,14 +38,151 @@ from src.agent_tools import (
 
 logger = logging.getLogger(__name__)
 
-_SENSITIVE_BEARER_RE = re.compile(
-    r"(?i)\b(authorization\s*[:=]\s*bearer\s+)[A-Za-z0-9._~+/=-]+"
+# Redaction patterns for common secret-bearing shapes. Explicit and tested
+# (see tests/test_loop_guard_signals.py) rather than one clever broad regex —
+# safety first, but we try not to mangle harmless prose. Applied in order.
+_REDACTED = "[redacted]"
+
+# Cookie: ... / Set-Cookie: ... — redact the rest of the line (cookies hold spaces).
+_SENSITIVE_COOKIE_RE = re.compile(
+    r"(?i)\b((?:set-)?cookie\s*[:=]\s*)[^\r\n]+"
 )
-_SENSITIVE_VALUE_RE = re.compile(
-    r"(?i)\b(password|passwd|pwd|token|api[_-]?key|secret)\b"
-    r"(\s*[:=]\s*)"
-    r"([^\s,;]+)"
+# URL credentials, e.g. postgres://user:pass@host/db. The password half allows
+# inner colons (postgres://user:pa:ss@host/db) but still stops at / and @.
+_SENSITIVE_URL_CRED_RE = re.compile(
+    r"(?i)\b([a-z][a-z0-9+.\-]*://)[^\s:/@]+:[^\s/@]+@"
 )
+# Prefix-only discovery regexes. Each matches the key and its separator (the part
+# we KEEP); the value that follows is found by a linear scanner rather than by a
+# regex, so there is no backtracking-prone quantifier over uncontrolled input.
+#
+# Authorization: Bearer <tok> / Authorization: Basic "two word secret"
+_AUTH_PREFIX_RE = re.compile(
+    r"(?i)authorization\s*[:=]\s*(?:bearer|basic)\s+"
+)
+# Provider-prefixed env names, e.g. OPENAI_API_KEY=..., AWS_SECRET_ACCESS_KEY=...,
+# GITHUB_TOKEN=... — require a sensitive suffix preceded by `_` so benign names
+# that merely end in KEY (MONKEY, TURKEY) are left alone.
+_ENV_PREFIX_RE = re.compile(
+    r"(?:export\s+)?\b[A-Z][A-Z0-9_]*"
+    r"_(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|PWD|CREDENTIALS?)\s*=\s*"
+)
+# Generic sensitive key, e.g. password=..., api_key: ..., client_secret=...
+_KEY_PREFIX_RE = re.compile(
+    r"(?i)\b(?:password|passwd|pwd|token|api[_-]?key|client_secret|secret)\b\s*[:=]\s*"
+)
+# Obvious provider-shaped bare tokens (no surrounding key needed).
+_SENSITIVE_BARE_TOKEN_RE = re.compile(
+    r"\b("
+    r"sk-[A-Za-z0-9_\-]{16,}"          # OpenAI / Anthropic style
+    r"|gh[pousr]_[A-Za-z0-9]{20,}"     # GitHub PAT
+    r"|xox[baprs]-[A-Za-z0-9\-]{10,}"  # Slack
+    r"|AKIA[0-9A-Z]{16}"               # AWS access key id
+    r"|hf_[A-Za-z0-9]{16,}"            # Hugging Face token
+    r"|AIza[0-9A-Za-z_\-]{20,}"        # Google API key
+    r")\b"
+)
+
+
+def _consume_secret_value_end(text: str, start: int) -> int:
+    """Return the exclusive end index of the secret value beginning at ``start``.
+
+    If the value is quoted, scan to the matching unescaped quote (backslash
+    escapes are skipped two chars at a time). Otherwise scan to the first
+    whitespace, comma, or semicolon. The scan is linear in the length of the
+    input, so it cannot exhibit catastrophic backtracking.
+    """
+    n = len(text)
+    if start >= n:
+        return start
+    quote = text[start]
+    if quote in ("'", '"'):
+        i = start + 1
+        while i < n:
+            ch = text[i]
+            if ch == "\\":
+                i += 2
+                continue
+            if ch == quote:
+                return i + 1
+            i += 1
+        return n  # unterminated quote: redact to the end
+    i = start
+    while i < n and not text[i].isspace() and text[i] not in (",", ";"):
+        i += 1
+    return i
+
+
+def _redact_after_prefix(text: str, prefix_re: "re.Pattern") -> str:
+    """Redact the value following each ``prefix_re`` match using a linear scan."""
+    result = []
+    pos = 0
+    n = len(text)
+    while pos < n:
+        match = prefix_re.search(text, pos)
+        if match is None:
+            result.append(text[pos:])
+            break
+        result.append(text[pos:match.end()])
+        value_end = _consume_secret_value_end(text, match.end())
+        if value_end > match.end():
+            result.append(_REDACTED)
+            pos = value_end
+        else:
+            # Empty value: nothing to redact; step past the prefix and continue.
+            pos = match.end()
+            if pos < n:
+                result.append(text[pos])
+                pos += 1
+    return "".join(result)
+
+
+def _redact_private_keys(text: str) -> str:
+    """Replace PEM private-key blocks with a placeholder via linear scanning.
+
+    Finds ``-----BEGIN `` markers, verifies the header names a PRIVATE KEY,
+    locates the matching ``-----END `` marker, and collapses the whole block.
+    No regex is used, so the (multi-line, uncontrolled) body cannot trigger
+    polynomial matching.
+    """
+    begin_marker = "-----BEGIN "
+    end_marker = "-----END "
+    dash = "-----"
+    max_header = 64  # generous bound on "[TYPE ]PRIVATE KEY"
+    result = []
+    pos = 0
+    while True:
+        begin = text.find(begin_marker, pos)
+        if begin == -1:
+            result.append(text[pos:])
+            return "".join(result)
+        header_start = begin + len(begin_marker)
+        header_close = text.find(dash, header_start)
+        if (
+            header_close == -1
+            or header_close - header_start > max_header
+            or not text[header_start:header_close].endswith("PRIVATE KEY")
+        ):
+            result.append(text[pos:header_start])
+            pos = header_start
+            continue
+        end = text.find(end_marker, header_close)
+        if end == -1:
+            result.append(text[pos:])
+            return "".join(result)
+        end_header_start = end + len(end_marker)
+        end_close = text.find(dash, end_header_start)
+        if (
+            end_close == -1
+            or end_close - end_header_start > max_header
+            or not text[end_header_start:end_close].endswith("PRIVATE KEY")
+        ):
+            result.append(text[pos:header_start])
+            pos = header_start
+            continue
+        result.append(text[pos:begin])
+        result.append("[redacted private key]")
+        pos = end_close + len(dash)
 
 
 def _redact_sensitive_text(value: object) -> str:
@@ -55,10 +191,13 @@ def _redact_sensitive_text(value: object) -> str:
         return ""
 
     text = str(value)
-    text = _SENSITIVE_BEARER_RE.sub(r"\1[redacted]", text)
-    return _SENSITIVE_VALUE_RE.sub(r"\1\2[redacted]", text)
-
-
+    text = _redact_private_keys(text)
+    text = _redact_after_prefix(text, _AUTH_PREFIX_RE)
+    text = _SENSITIVE_COOKIE_RE.sub(r"\1" + _REDACTED, text)
+    text = _SENSITIVE_URL_CRED_RE.sub(r"\1" + _REDACTED + "@", text)
+    text = _redact_after_prefix(text, _ENV_PREFIX_RE)
+    text = _redact_after_prefix(text, _KEY_PREFIX_RE)
+    return _SENSITIVE_BARE_TOKEN_RE.sub(_REDACTED, text)
 
 
 def _load_mcp_disabled_map() -> Dict[str, set]:
@@ -2670,7 +2809,12 @@ async def stream_agent_loop(
             if _looks_like_promise:
                 _intent_nudge_count += 1
                 _matched_phrase = _intent_match.group(0).strip()
-                logger.info(f"[agent] intent-without-action nudge #{_intent_nudge_count} on round {round_num}: {_matched_phrase!r}")
+                # Don't log the matched phrase — it's raw model text that may
+                # carry credentials. Structural metadata only.
+                logger.info(
+                    "[agent] intent-without-action nudge #%d on round %d",
+                    _intent_nudge_count, round_num,
+                )
                 messages.append({
                     "role": "system",
                     "content": (
@@ -2691,14 +2835,17 @@ async def stream_agent_loop(
             # look like a clean completion.
             if _promise_shape and _intent_nudge_count >= _MAX_INTENT_NUDGES:
                 _matched_phrase = _intent_match.group(0).strip()
+                _matched_phrase_safe = _redact_sensitive_text(_matched_phrase)
                 _in_message = (
                     f"Intent-nudge cap reached on round {round_num}: the model "
-                    f"announced an action ({_matched_phrase!r}) without a tool call "
+                    f"announced an action ({_matched_phrase_safe!r}) without a tool call "
                     f"after {_intent_nudge_count} nudge(s); ending the turn."
                 )
+                # Do not log the matched phrase, even redacted. It is raw model
+                # text and may contain credentials; keep logs structural only.
                 logger.warning(
-                    "[agent] intent-nudge cap exhausted on round %d (%d/%d): %r",
-                    round_num, _intent_nudge_count, _MAX_INTENT_NUDGES, _matched_phrase,
+                    "[agent] intent-nudge cap exhausted on round %d (%d/%d)",
+                    round_num, _intent_nudge_count, _MAX_INTENT_NUDGES,
                 )
                 yield f'data: {json.dumps({"type": "intent_nudge_exhausted", "round": round_num, "nudges": _intent_nudge_count, "max_nudges": _MAX_INTENT_NUDGES, "message": _in_message})}\n\n'
             break  # no tools — done
@@ -2740,10 +2887,12 @@ async def stream_agent_loop(
                 f"Loop-breaker stopped the agent on round {round_num}: {reason}. "
                 "Forced one tool-free round to converge on an answer or state what's blocked."
             )
+            # Log structural metadata only — `_sig` is raw tool-call content
+            # that may carry credentials.
             logger.warning(
                 "[agent] loop-breaker tripped on round %d (%s); "
-                "stuck_rounds=%d/%d runaway=%r sig=%r",
-                round_num, reason, _stuck_rounds, _MAX_STUCK_ROUNDS, _runaway, _sig[:80],
+                "stuck_rounds=%d/%d runaway=%r",
+                round_num, reason, _stuck_rounds, _MAX_STUCK_ROUNDS, _runaway,
             )
             # Surface the stop cause to the stream so the user (and journalctl)
             # can tell a guard fired, not a clean completion.
@@ -2826,6 +2975,10 @@ async def stream_agent_loop(
                 cmd_display = block.content.split("\n")[0].strip()[:80]
             else:
                 cmd_display = block.content.strip()
+            # The display string is streamed (tool_start/tool_output) and persisted;
+            # redact any secrets in it. block.content itself is left untouched so
+            # tool execution still sees the real command.
+            cmd_display = _redact_sensitive_text(cmd_display)
 
             if tool_policy and tool_policy.blocks(block.tool_type):
                 desc = f"{block.tool_type}: BLOCKED"
@@ -2871,8 +3024,15 @@ async def stream_agent_loop(
                     evt = await _progress_q.get()
                     if evt is None:
                         break
+                    # Redact secrets in the live tail before streaming — the
+                    # final tool_output is redacted, so the progress tail must
+                    # be too, or a secret could flash by mid-run. Copy so we
+                    # don't mutate the tool's own event payload.
+                    _evt = dict(evt)
+                    if isinstance(_evt.get("tail"), str):
+                        _evt["tail"] = _redact_sensitive_text(_evt["tail"])
                     yield (
-                        f'data: {json.dumps({"type": "tool_progress", "tool": block.tool_type, "round": round_num, **evt})}\n\n'
+                        f'data: {json.dumps({"type": "tool_progress", "tool": block.tool_type, "round": round_num, **_evt})}\n\n'
                     )
                 desc, result = await _tool_task
 
@@ -2937,6 +3097,72 @@ async def stream_agent_loop(
                             elif "results" in result:
                                 result["results"] = _clean
                             elif "stdout" in result:
+                                result["stdout"] = _clean
+                        except Exception:
+                            pass
+
+            # Emit doc-specific event for document tools — the frontend
+            # document panel handles this; no need to show content in chat.
+            if is_doc_tool and "action" in result:
+                if result["action"] == "suggest":
+                    yield (
+                        f'data: {json.dumps({"type": "doc_suggestions", "doc_id": result["doc_id"], "suggestions": result["suggestions"]})}\n\n'
+                    )
+                else:
+                    yield (
+                        f'data: {json.dumps({"type": "doc_update", "doc_id": result["doc_id"], "content": result["content"], "version": result["version"], "title": result.get("title", ""), "language": result.get("language")})}\n\n'
+                    )
+
+            # Emit ui_control event for frontend to apply UI changes
+            if "ui_event" in result:
+                yield (
+                    f'data: {json.dumps({"type": "ui_control", "data": result})}\n\n'
+                )
+
+            # ask_user: the agent posed a multiple-choice question. Emit it so the
+            # frontend renders clickable options, then end the turn (below) and
+            # wait — the user's pick becomes the next message.
+            if "ask_user" in result:
+                # The question lives in the tool args. ChatMessage.to_dict()
+                # replays only role+content to the model next turn — tool_event
+                # metadata is dropped — so if the question is never in the saved
+                # assistant text, the model can't see it already asked and will
+                # loop and re-ask after the user answers. Stream it as assistant
+                # text (once) so it persists and is replayed. The card shows the
+                # options only, so this is the single visible copy of the question.
+                _auq = result["ask_user"]
+                _auq_q = (_auq.get("question") or "").strip()
+                if _auq_q and _auq_q not in full_response:
+                    _auq_delta = ("\n\n" if full_response.strip() else "") + _auq_q
+                    full_response += _auq_delta
+                    yield 'data: ' + json.dumps({"delta": _auq_delta}) + '\n\n'
+                yield (
+                    f'data: {json.dumps({"type": "ask_user", "data": result["ask_user"]})}\n\n'
+                )
+                _awaiting_user = True
+
+            # update_plan: agent wrote back to the plan (ticked a step / revised).
+            # Push it to the frontend so the stored plan + docked window update
+            # live. Does NOT end the turn — the agent keeps working.
+            if "plan_update" in result:
+                yield (
+                    f'data: {json.dumps({"type": "plan_update", "data": result["plan_update"]})}\n\n'
+                )
+
+            # Build output for frontend tool bubble.
+            # Document tools get a short summary — content goes to the editor panel.
+            output_text = ""
+            if is_doc_tool and "action" in result:
+                action = result["action"]
+                title = result.get("title", "")
+                ver = result.get("version", "?")
+                if action == "create":
+                    output_text = f'Document created: "{title}" (v{ver})'
+                elif action == "edit":
+                    output_text = f'Document edited: "{title}" (v{ver}, {result.get("applied", 0)} edit(s))'
+                elif action == "update":
+                    output_text = f'Document updated: "{title}" (v{ver})'
+            elif "stdout" in result:
                 # On a bash/python timeout the result carries error + (often
                 # empty) stdout/stderr; fall back to the error so the "timed
                 # out" reason reaches the UI instead of a blank result.