fix(model-context): read real context window for unknown proxy models (#4909)

api/proxy endpoints (OpenRouter, other OpenAI-compatible aggregators)
short-circuit _query_context_length: they only consult the static
KNOWN_CONTEXT_WINDOWS table and otherwise return DEFAULT_CONTEXT (128000).
Any model not in that table — e.g. a freshly listed OpenRouter model like
Owl-alpha — was therefore capped at 128k even though the endpoint's catalog
reports its true window (1048576), so the rest of the model context never
got used.

The short-circuit exists so a context lookup doesn't download a large proxy
catalog on every call. Keep that property for the common case: known models
still resolve from the table with no network. For a model missing from the
table, read the window from the endpoint's /models catalog and cache the
whole id->context map per endpoint, so the catalog is fetched at most once
per endpoint (not once per model) and only for models that were broken
anyway. On any fetch/parse failure or a model absent from the catalog, fall
back to DEFAULT_CONTEXT exactly as before.

Factor the per-entry field extraction the non-proxy path already used into
_model_ctx_from_entry so both paths share it.

Fixes #4886

src/model_context.py

@@ -316,6 +316,83 @@ def _lookup_known(model: str) -> Optional[int]:
     return best_ctx
 
 
+def _model_ctx_from_entry(m: dict) -> Optional[int]:
+    """Extract a positive context window from one /models catalog entry.
+
+    Checks the common top-level fields first, then a nested meta/model_extra
+    object. Returns None when no positive window is reported.
+    """
+    if not isinstance(m, dict):
+        return None
+    for field in (
+        "context_length",
+        "context_window",
+        "max_model_len",
+        "max_context_length",
+        "max_seq_len",
+    ):
+        val = m.get(field)
+        if val and isinstance(val, (int, float)) and val > 0:
+            return int(val)
+    meta = m.get("meta") or m.get("model_extra") or {}
+    if isinstance(meta, dict):
+        # n_ctx is the actual serving context (set via -c flag in llama.cpp)
+        for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
+            val = meta.get(field)
+            if val and isinstance(val, (int, float)) and val > 0:
+                return int(val)
+    return None
+
+
+# Per-endpoint cache of the {model_id: context_length} map parsed from a
+# proxy/api catalog. api/proxy endpoints skip the /models download on every
+# lookup because a large catalog is expensive; caching the whole map lets us
+# pay that download at most once per endpoint instead of once per model.
+_catalog_ctx_cache: Dict[str, Dict[str, int]] = {}
+
+
+def _proxy_catalog_context(endpoint_url: str, model: str) -> Optional[int]:
+    """Context window for a model read from the endpoint's /models catalog.
+
+    Fetches the catalog once per endpoint and caches the full id->context map,
+    so an api/proxy endpoint serving a model that isn't in KNOWN_CONTEXT_WINDOWS
+    (e.g. a new OpenRouter model) still reports its real window instead of the
+    bare default. Returns None when the catalog can't be read or doesn't list a
+    positive window for the model.
+    """
+    cat = _catalog_ctx_cache.get(endpoint_url)
+    if cat is None:
+        from src.endpoint_resolver import build_models_url
+        try:
+            r = httpx.get(build_models_url(endpoint_url), timeout=REQUEST_TIMEOUT)
+        except Exception as e:
+            logger.debug(f"Failed to fetch proxy catalog for context length: {e}")
+            return None
+        if not r.is_success:
+            return None
+        cat = {}
+        try:
+            for m in (r.json().get("data") or []):
+                mid = m.get("id") if isinstance(m, dict) else None
+                ctx = _model_ctx_from_entry(m) if mid else None
+                if mid and ctx:
+                    cat[mid] = ctx
+        except Exception as e:
+            logger.debug(f"Failed to parse proxy catalog for context length: {e}")
+            return None
+        _catalog_ctx_cache[endpoint_url] = cat
+
+    if model in cat:
+        return cat[model]
+    # Catalog ids may carry a provider prefix (e.g. "openai/gpt-4o") while the
+    # session stores the bare id; match on the trailing segment as a fallback.
+    base = model.split("/")[-1]
+    for mid, ctx in cat.items():
+        if mid.split("/")[-1] == base:
+            return ctx
+    return None
+
+
 def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
     """Query the model API for context length. Returns (context_length, known) where
     ``known`` is False only for the bare DEFAULT_CONTEXT fallback."""
@@ -330,6 +407,14 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
         if known:
             logger.info(f"Using known context window for {model}: {known}")
             return known, True
+        # Not in the known table: read the real window from the catalog (cached
+        # once per endpoint) instead of capping every unknown model at the
+        # default — that under-reported large windows on aggregators like
+        # OpenRouter (issue #4886).
+        api_ctx = _proxy_catalog_context(endpoint_url, model)
+        if api_ctx:
+            logger.info(f"Proxy catalog reports context window for {model}: {api_ctx}")
+            return api_ctx, True
         return DEFAULT_CONTEXT, False
 
     # Try llama.cpp /slots endpoint first — reports actual serving context
@@ -370,27 +455,7 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]:
             for m in models_list:
                 mid = m.get("id", "")
                 if mid == model or mid.split("/")[-1] == model.split("/")[-1]:
-                    for field in (
-                        "context_length",
-                        "context_window",
-                        "max_model_len",
-                        "max_context_length",
-                        "max_seq_len",
-                    ):
-                        val = m.get(field)
-                        if val and isinstance(val, (int, float)) and val > 0:
-                            api_ctx = int(val)
-                            break
-
-                    if not api_ctx:
-                        meta = m.get("meta") or m.get("model_extra") or {}
-                        if isinstance(meta, dict):
-                            # n_ctx is the actual serving context (set via -c flag in llama.cpp)
-                            for field in ("n_ctx", "context_length", "context_window", "max_model_len"):
-                                val = meta.get(field)
-                                if val and isinstance(val, (int, float)) and val > 0:
-                                    api_ctx = int(val)
-                                    break
+                    api_ctx = _model_ctx_from_entry(m)
                     break
     except Exception as e:
         logger.debug(f"Failed to query context length for {model}: {e}")

tests/test_model_context.py

@@ -191,9 +191,19 @@ class TestLookupKnown:
         assert _lookup_known("gpt-4") == 8192
 
 
+class _FakeResp:
+    def __init__(self, payload, ok=True):
+        self._payload = payload
+        self.is_success = ok
+
+    def json(self):
+        return self._payload
+
+
 class TestGetContextLength:
     def setup_method(self):
         model_context._context_cache.clear()
+        model_context._catalog_ctx_cache.clear()
 
     def test_local_endpoint_requeries_same_model_after_restart(self, monkeypatch):
         calls = []
@@ -233,7 +243,7 @@ class TestGetContextLength:
         assert second == 200000
         assert len(calls) == 1
 
-    def test_configured_proxy_uses_default_without_model_listing(self, monkeypatch):
+    def _proxy_db(self, monkeypatch):
         _install_endpoint_db(monkeypatch, [
             types.SimpleNamespace(
                 base_url="http://100.117.136.97:34521/v1",
@@ -242,18 +252,63 @@ class TestGetContextLength:
                 is_enabled=True,
             )
         ])
-        calls = []
+
+    def test_configured_proxy_known_model_skips_model_listing(self, monkeypatch):
+        # A model covered by the known-context table must still resolve without
+        # touching /models — the cheap path the proxy short-circuit exists for.
+        self._proxy_db(monkeypatch)
 
         def fake_get(*args, **kwargs):
-            calls.append(args)
-            raise AssertionError("/models should not be queried for configured proxy context")
+            raise AssertionError("/models must not be queried for a known proxy model")
 
         monkeypatch.setattr(model_context.httpx, "get", fake_get)
 
         endpoint = "http://100.117.136.97:34521/v1/chat/completions"
-        first = model_context.get_context_length(endpoint, "unknown-proxy-model")
-        second = model_context.get_context_length(endpoint, "unknown-proxy-model")
+        assert model_context.get_context_length(endpoint, "gpt-4o") == 128000
 
-        assert first == model_context.DEFAULT_CONTEXT
-        assert second == model_context.DEFAULT_CONTEXT
-        assert calls == []
+    def test_configured_proxy_unknown_model_reads_catalog_context(self, monkeypatch):
+        # A model missing from the known table (e.g. a new OpenRouter model)
+        # must report the catalog's real window, not the bare default (#4886).
+        # The catalog is fetched once per endpoint and reused for other models.
+        self._proxy_db(monkeypatch)
+        fetches = []
+
+        def fake_get(url, *args, **kwargs):
+            fetches.append(url)
+            return _FakeResp({"data": [
+                {"id": "owl-alpha", "context_length": 1048576},
+                {"id": "tiny-proxy-model", "context_length": 8192},
+            ]})
+
+        monkeypatch.setattr(model_context.httpx, "get", fake_get)
+
+        endpoint = "http://100.117.136.97:34521/v1/chat/completions"
+        assert model_context.get_context_length(endpoint, "owl-alpha") == 1048576
+        # A second unknown model on the same endpoint reuses the cached catalog.
+        assert model_context.get_context_length(endpoint, "tiny-proxy-model") == 8192
+        assert len(fetches) == 1
+
+    def test_configured_proxy_unknown_model_falls_back_to_default(self, monkeypatch):
+        # If the catalog can be read but doesn't list the model, keep the
+        # conservative default rather than guessing.
+        self._proxy_db(monkeypatch)
+
+        def fake_get(url, *args, **kwargs):
+            return _FakeResp({"data": [{"id": "some-other-model", "context_length": 4096}]})
+
+        monkeypatch.setattr(model_context.httpx, "get", fake_get)
+
+        endpoint = "http://100.117.136.97:34521/v1/chat/completions"
+        assert model_context.get_context_length(endpoint, "absent-model") == model_context.DEFAULT_CONTEXT
+
+    def test_configured_proxy_catalog_fetch_failure_uses_default(self, monkeypatch):
+        # A failed/unreachable catalog must not raise — fall back to the default.
+        self._proxy_db(monkeypatch)
+
+        def fake_get(url, *args, **kwargs):
+            raise RuntimeError("network down")
+
+        monkeypatch.setattr(model_context.httpx, "get", fake_get)
+
+        endpoint = "http://100.117.136.97:34521/v1/chat/completions"
+        assert model_context.get_context_length(endpoint, "unknown-proxy-model") == model_context.DEFAULT_CONTEXT