From 9a80ab24afc9125e83b69269a2e07c529f15a375 Mon Sep 17 00:00:00 2001 From: Ashvin <76151462+ashvinctrl@users.noreply.github.com> Date: Tue, 30 Jun 2026 22:34:29 +0530 Subject: [PATCH] fix(model-context): read real context window for unknown proxy models (#4909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit api/proxy endpoints (OpenRouter, other OpenAI-compatible aggregators) short-circuit _query_context_length: they only consult the static KNOWN_CONTEXT_WINDOWS table and otherwise return DEFAULT_CONTEXT (128000). Any model not in that table — e.g. a freshly listed OpenRouter model like Owl-alpha — was therefore capped at 128k even though the endpoint's catalog reports its true window (1048576), so the rest of the model context never got used. The short-circuit exists so a context lookup doesn't download a large proxy catalog on every call. Keep that property for the common case: known models still resolve from the table with no network. For a model missing from the table, read the window from the endpoint's /models catalog and cache the whole id->context map per endpoint, so the catalog is fetched at most once per endpoint (not once per model) and only for models that were broken anyway. On any fetch/parse failure or a model absent from the catalog, fall back to DEFAULT_CONTEXT exactly as before. Factor the per-entry field extraction the non-proxy path already used into _model_ctx_from_entry so both paths share it. Fixes #4886 --- src/model_context.py | 107 +++++++++++++++++++++++++++++------- tests/test_model_context.py | 73 +++++++++++++++++++++--- 2 files changed, 150 insertions(+), 30 deletions(-) diff --git a/src/model_context.py b/src/model_context.py index 342047282..b6afa801e 100644 --- a/src/model_context.py +++ b/src/model_context.py @@ -316,6 +316,83 @@ def _lookup_known(model: str) -> Optional[int]: return best_ctx +def _model_ctx_from_entry(m: dict) -> Optional[int]: + """Extract a positive context window from one /models catalog entry. + + Checks the common top-level fields first, then a nested meta/model_extra + object. Returns None when no positive window is reported. + """ + if not isinstance(m, dict): + return None + for field in ( + "context_length", + "context_window", + "max_model_len", + "max_context_length", + "max_seq_len", + ): + val = m.get(field) + if val and isinstance(val, (int, float)) and val > 0: + return int(val) + meta = m.get("meta") or m.get("model_extra") or {} + if isinstance(meta, dict): + # n_ctx is the actual serving context (set via -c flag in llama.cpp) + for field in ("n_ctx", "context_length", "context_window", "max_model_len"): + val = meta.get(field) + if val and isinstance(val, (int, float)) and val > 0: + return int(val) + return None + + +# Per-endpoint cache of the {model_id: context_length} map parsed from a +# proxy/api catalog. api/proxy endpoints skip the /models download on every +# lookup because a large catalog is expensive; caching the whole map lets us +# pay that download at most once per endpoint instead of once per model. +_catalog_ctx_cache: Dict[str, Dict[str, int]] = {} + + +def _proxy_catalog_context(endpoint_url: str, model: str) -> Optional[int]: + """Context window for a model read from the endpoint's /models catalog. + + Fetches the catalog once per endpoint and caches the full id->context map, + so an api/proxy endpoint serving a model that isn't in KNOWN_CONTEXT_WINDOWS + (e.g. a new OpenRouter model) still reports its real window instead of the + bare default. Returns None when the catalog can't be read or doesn't list a + positive window for the model. + """ + cat = _catalog_ctx_cache.get(endpoint_url) + if cat is None: + from src.endpoint_resolver import build_models_url + try: + r = httpx.get(build_models_url(endpoint_url), timeout=REQUEST_TIMEOUT) + except Exception as e: + logger.debug(f"Failed to fetch proxy catalog for context length: {e}") + return None + if not r.is_success: + return None + cat = {} + try: + for m in (r.json().get("data") or []): + mid = m.get("id") if isinstance(m, dict) else None + ctx = _model_ctx_from_entry(m) if mid else None + if mid and ctx: + cat[mid] = ctx + except Exception as e: + logger.debug(f"Failed to parse proxy catalog for context length: {e}") + return None + _catalog_ctx_cache[endpoint_url] = cat + + if model in cat: + return cat[model] + # Catalog ids may carry a provider prefix (e.g. "openai/gpt-4o") while the + # session stores the bare id; match on the trailing segment as a fallback. + base = model.split("/")[-1] + for mid, ctx in cat.items(): + if mid.split("/")[-1] == base: + return ctx + return None + + def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]: """Query the model API for context length. Returns (context_length, known) where ``known`` is False only for the bare DEFAULT_CONTEXT fallback.""" @@ -330,6 +407,14 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]: if known: logger.info(f"Using known context window for {model}: {known}") return known, True + # Not in the known table: read the real window from the catalog (cached + # once per endpoint) instead of capping every unknown model at the + # default — that under-reported large windows on aggregators like + # OpenRouter (issue #4886). + api_ctx = _proxy_catalog_context(endpoint_url, model) + if api_ctx: + logger.info(f"Proxy catalog reports context window for {model}: {api_ctx}") + return api_ctx, True return DEFAULT_CONTEXT, False # Try llama.cpp /slots endpoint first — reports actual serving context @@ -370,27 +455,7 @@ def _query_context_length(endpoint_url: str, model: str) -> Tuple[int, bool]: for m in models_list: mid = m.get("id", "") if mid == model or mid.split("/")[-1] == model.split("/")[-1]: - for field in ( - "context_length", - "context_window", - "max_model_len", - "max_context_length", - "max_seq_len", - ): - val = m.get(field) - if val and isinstance(val, (int, float)) and val > 0: - api_ctx = int(val) - break - - if not api_ctx: - meta = m.get("meta") or m.get("model_extra") or {} - if isinstance(meta, dict): - # n_ctx is the actual serving context (set via -c flag in llama.cpp) - for field in ("n_ctx", "context_length", "context_window", "max_model_len"): - val = meta.get(field) - if val and isinstance(val, (int, float)) and val > 0: - api_ctx = int(val) - break + api_ctx = _model_ctx_from_entry(m) break except Exception as e: logger.debug(f"Failed to query context length for {model}: {e}") diff --git a/tests/test_model_context.py b/tests/test_model_context.py index ad752fea7..86872f30d 100644 --- a/tests/test_model_context.py +++ b/tests/test_model_context.py @@ -191,9 +191,19 @@ class TestLookupKnown: assert _lookup_known("gpt-4") == 8192 +class _FakeResp: + def __init__(self, payload, ok=True): + self._payload = payload + self.is_success = ok + + def json(self): + return self._payload + + class TestGetContextLength: def setup_method(self): model_context._context_cache.clear() + model_context._catalog_ctx_cache.clear() def test_local_endpoint_requeries_same_model_after_restart(self, monkeypatch): calls = [] @@ -233,7 +243,7 @@ class TestGetContextLength: assert second == 200000 assert len(calls) == 1 - def test_configured_proxy_uses_default_without_model_listing(self, monkeypatch): + def _proxy_db(self, monkeypatch): _install_endpoint_db(monkeypatch, [ types.SimpleNamespace( base_url="http://100.117.136.97:34521/v1", @@ -242,18 +252,63 @@ class TestGetContextLength: is_enabled=True, ) ]) - calls = [] + + def test_configured_proxy_known_model_skips_model_listing(self, monkeypatch): + # A model covered by the known-context table must still resolve without + # touching /models — the cheap path the proxy short-circuit exists for. + self._proxy_db(monkeypatch) def fake_get(*args, **kwargs): - calls.append(args) - raise AssertionError("/models should not be queried for configured proxy context") + raise AssertionError("/models must not be queried for a known proxy model") monkeypatch.setattr(model_context.httpx, "get", fake_get) endpoint = "http://100.117.136.97:34521/v1/chat/completions" - first = model_context.get_context_length(endpoint, "unknown-proxy-model") - second = model_context.get_context_length(endpoint, "unknown-proxy-model") + assert model_context.get_context_length(endpoint, "gpt-4o") == 128000 - assert first == model_context.DEFAULT_CONTEXT - assert second == model_context.DEFAULT_CONTEXT - assert calls == [] + def test_configured_proxy_unknown_model_reads_catalog_context(self, monkeypatch): + # A model missing from the known table (e.g. a new OpenRouter model) + # must report the catalog's real window, not the bare default (#4886). + # The catalog is fetched once per endpoint and reused for other models. + self._proxy_db(monkeypatch) + fetches = [] + + def fake_get(url, *args, **kwargs): + fetches.append(url) + return _FakeResp({"data": [ + {"id": "owl-alpha", "context_length": 1048576}, + {"id": "tiny-proxy-model", "context_length": 8192}, + ]}) + + monkeypatch.setattr(model_context.httpx, "get", fake_get) + + endpoint = "http://100.117.136.97:34521/v1/chat/completions" + assert model_context.get_context_length(endpoint, "owl-alpha") == 1048576 + # A second unknown model on the same endpoint reuses the cached catalog. + assert model_context.get_context_length(endpoint, "tiny-proxy-model") == 8192 + assert len(fetches) == 1 + + def test_configured_proxy_unknown_model_falls_back_to_default(self, monkeypatch): + # If the catalog can be read but doesn't list the model, keep the + # conservative default rather than guessing. + self._proxy_db(monkeypatch) + + def fake_get(url, *args, **kwargs): + return _FakeResp({"data": [{"id": "some-other-model", "context_length": 4096}]}) + + monkeypatch.setattr(model_context.httpx, "get", fake_get) + + endpoint = "http://100.117.136.97:34521/v1/chat/completions" + assert model_context.get_context_length(endpoint, "absent-model") == model_context.DEFAULT_CONTEXT + + def test_configured_proxy_catalog_fetch_failure_uses_default(self, monkeypatch): + # A failed/unreachable catalog must not raise — fall back to the default. + self._proxy_db(monkeypatch) + + def fake_get(url, *args, **kwargs): + raise RuntimeError("network down") + + monkeypatch.setattr(model_context.httpx, "get", fake_get) + + endpoint = "http://100.117.136.97:34521/v1/chat/completions" + assert model_context.get_context_length(endpoint, "unknown-proxy-model") == model_context.DEFAULT_CONTEXT