feat(models): define capability schema and readers (#2739)

* feat(models): define capability schema and readers

* fix(models): harden Google catalog probing

Restrict native catalog probing to the Gemini host, keep provider keys out of request URLs, filter non-chat model resources, and preserve the manual refresh default in the built-in Google add flow.
This commit is contained in:
RaresKeY
2026-07-18 10:40:58 +02:00
committed by GitHub
parent f87107e16f
commit 4f04c347cc
17 changed files with 3861 additions and 4 deletions
+115 -3
View File
@@ -472,7 +472,11 @@ def _endpoint_kind(ep: Any) -> str:
def _endpoint_refresh_mode(ep: Any, endpoint_kind: str | None = None) -> str: def _endpoint_refresh_mode(ep: Any, endpoint_kind: str | None = None) -> str:
return _normalize_refresh_mode(getattr(ep, "model_refresh_mode", None), endpoint_kind or _endpoint_kind(ep)) return _normalize_endpoint_refresh_mode(
getattr(ep, "model_refresh_mode", None),
endpoint_kind or _endpoint_kind(ep),
getattr(ep, "base_url", ""),
)
def _endpoint_refresh_interval(ep: Any, category: str) -> float: def _endpoint_refresh_interval(ep: Any, category: str) -> float:
@@ -851,6 +855,99 @@ def _ollama_model_names(data: Any) -> List[str]:
return out return out
def _is_google_api_base(base_url: str) -> bool:
try:
return (urlparse(base_url).hostname or "").lower() == "generativelanguage.googleapis.com"
except Exception:
return False
def _normalize_endpoint_refresh_mode(value: Any, endpoint_kind: str = "auto", base_url: str = "") -> str:
if not str(value or "").strip() and _is_google_api_base(base_url):
return "manual"
return _normalize_refresh_mode(value, endpoint_kind)
def _google_native_root(base_url: str) -> str:
"""Return the Gemini native API root for a Google endpoint.
Chat calls may be configured against Google's OpenAI-compatible
`/openai` path, but model catalog reads should use the native Models API
so we get Google's current Model resource shape.
"""
try:
parsed = urlparse(base_url)
except Exception:
return "https://generativelanguage.googleapis.com/v1beta"
path = (parsed.path or "").rstrip("/")
if path.endswith("/openai"):
path = path[: -len("/openai")].rstrip("/")
if not path:
path = "/v1beta"
return urlunparse(parsed._replace(path=path, query="", fragment="")).rstrip("/")
def _google_native_models_url(base_url: str) -> str:
return _google_native_root(base_url) + "/models"
def _google_model_id_from_item(item: Any) -> str:
if not isinstance(item, dict):
return ""
value = item.get("baseModelId") or item.get("name") or item.get("model") or ""
return str(value or "").strip().removeprefix("models/")
def _google_model_supports_chat(item: Any) -> bool:
"""Return whether a native Google Model resource supports chat generation."""
if not isinstance(item, dict):
return False
methods = item.get("supportedGenerationMethods")
if not isinstance(methods, list):
return False
chat_methods = {"generateContent", "generateMessage", "generateText", "generateAnswer"}
return any(method in chat_methods for method in methods)
def _probe_google_models(base_url: str, api_key: str = None, timeout: int = 5, page_size: int = 1000) -> List[str]:
"""Read Google's native paginated Models API.
This intentionally returns only provider-reported model IDs. Capability
mapping is handled by the model capability reader and must not infer from
names here.
"""
url = _google_native_models_url(base_url)
try:
page_size = min(max(int(page_size or 1000), 1), 1000)
except Exception:
page_size = 1000
headers = {"Accept": "application/json"}
if api_key:
headers["x-goog-api-key"] = api_key
params: Dict[str, Any] = {"pageSize": page_size}
models: List[str] = []
seen = set()
page_token = ""
for _ in range(20):
request_params = dict(params)
if page_token:
request_params["pageToken"] = page_token
r = httpx.get(url, headers=headers, params=request_params, timeout=timeout, verify=llm_verify())
r.raise_for_status()
data = r.json()
for item in data.get("models") or []:
if not _google_model_supports_chat(item):
continue
model_id = _google_model_id_from_item(item)
if model_id and model_id not in seen:
seen.add(model_id)
models.append(model_id)
page_token = str(data.get("nextPageToken") or "").strip()
if not page_token:
break
return models
def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]: def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]:
"""Probe a base URL's /models endpoint and return list of model IDs. """Probe a base URL's /models endpoint and return list of model IDs.
For Anthropic, queries their /v1/models API, falling back to hardcoded list.""" For Anthropic, queries their /v1/models API, falling back to hardcoded list."""
@@ -863,6 +960,17 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
if api_key: if api_key:
return fetch_available_models(api_key, timeout=timeout) return fetch_available_models(api_key, timeout=timeout)
return [] return []
if _is_google_api_base(base):
try:
models = _probe_google_models(base, api_key, timeout=timeout)
if models:
return models
except httpx.HTTPStatusError as e:
status = e.response.status_code if e.response is not None else "unknown"
logger.warning(f"Google native models probe failed: HTTP {status}")
except Exception as e:
logger.warning(f"Google native models probe failed: {e}")
return []
if provider == "anthropic": if provider == "anthropic":
# Try Anthropic's /v1/models endpoint first # Try Anthropic's /v1/models endpoint first
url = _safe_build_models_url(base) url = _safe_build_models_url(base)
@@ -1854,7 +1962,7 @@ def setup_model_routes(model_discovery):
name = base_url.replace("http://", "").replace("https://", "").split("/")[0] name = base_url.replace("http://", "").replace("https://", "").split("/")[0]
requested_kind = _normalize_endpoint_kind(endpoint_kind) requested_kind = _normalize_endpoint_kind(endpoint_kind)
refresh_mode = _normalize_refresh_mode(model_refresh_mode, requested_kind) refresh_mode = _normalize_endpoint_refresh_mode(model_refresh_mode, requested_kind, base_url)
refresh_interval = _parse_positive_int(model_refresh_interval, minimum=30, maximum=86400) refresh_interval = _parse_positive_int(model_refresh_interval, minimum=30, maximum=86400)
refresh_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60) refresh_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60)
require_model_list = _truthy(require_models) require_model_list = _truthy(require_models)
@@ -2364,7 +2472,11 @@ def setup_model_routes(model_discovery):
if "endpoint_kind" in body: if "endpoint_kind" in body:
ep.endpoint_kind = _normalize_endpoint_kind(body.get("endpoint_kind")) ep.endpoint_kind = _normalize_endpoint_kind(body.get("endpoint_kind"))
if "model_refresh_mode" in body: if "model_refresh_mode" in body:
ep.model_refresh_mode = _normalize_refresh_mode(body.get("model_refresh_mode"), _endpoint_kind(ep)) ep.model_refresh_mode = _normalize_endpoint_refresh_mode(
body.get("model_refresh_mode"),
_endpoint_kind(ep),
ep.base_url,
)
if "model_refresh_interval" in body: if "model_refresh_interval" in body:
interval = _parse_positive_int(body.get("model_refresh_interval"), minimum=30, maximum=86400) interval = _parse_positive_int(body.get("model_refresh_interval"), minimum=30, maximum=86400)
ep.model_refresh_interval = interval ep.model_refresh_interval = interval
+934
View File
@@ -0,0 +1,934 @@
"""Canonical model capability metadata helpers.
This module defines shape and normalization only. It does not probe providers,
change routing, or infer authoritative capabilities from a bare model ID.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
FAMILY_CHAT = "chat"
FAMILY_EMBEDDING = "embedding"
FAMILY_IMAGE = "image"
FAMILY_VIDEO = "video"
FAMILY_AUDIO = "audio"
FAMILY_RERANK = "rerank"
FAMILY_CLASSIFICATION = "classification"
FAMILY_MODERATION = "moderation"
FAMILY_UNKNOWN = "unknown"
FAMILIES = frozenset(
{
FAMILY_CHAT,
FAMILY_EMBEDDING,
FAMILY_IMAGE,
FAMILY_VIDEO,
FAMILY_AUDIO,
FAMILY_RERANK,
FAMILY_CLASSIFICATION,
FAMILY_MODERATION,
FAMILY_UNKNOWN,
}
)
MODALITY_TEXT = "text"
MODALITY_IMAGE = "image"
MODALITY_FILE = "file"
MODALITY_PDF = "pdf"
MODALITY_AUDIO = "audio"
MODALITY_VIDEO = "video"
MODALITY_EMBEDDING = "embedding"
MODALITIES = frozenset(
{
MODALITY_TEXT,
MODALITY_IMAGE,
MODALITY_FILE,
MODALITY_PDF,
MODALITY_AUDIO,
MODALITY_VIDEO,
MODALITY_EMBEDDING,
}
)
CAP_VISION = "vision"
CAP_FILES = "files"
CAP_PDF = "pdf"
CAP_AUDIO_INPUT = "audio_input"
CAP_AUDIO_OUTPUT = "audio_output"
CAP_IMAGE_GENERATION = "image_generation"
CAP_IMAGE_EDITING = "image_editing"
CAP_INPAINTING = "inpainting"
CAP_VIDEO_GENERATION = "video_generation"
CAP_REASONING = "reasoning"
CAP_TOOL_CALL = "tool_call"
CAP_STRUCTURED_OUTPUT = "structured_output"
CAP_WEB_SEARCH = "web_search"
CAP_STREAMING = "streaming"
CAP_JSON_MODE = "json_mode"
CAP_TRANSCRIPTION = "transcription"
CAP_TTS = "tts"
CAP_REALTIME = "realtime"
CAP_TEXT_RENDERING = "text_rendering"
CAPABILITIES = frozenset(
{
CAP_VISION,
CAP_FILES,
CAP_PDF,
CAP_AUDIO_INPUT,
CAP_AUDIO_OUTPUT,
CAP_IMAGE_GENERATION,
CAP_IMAGE_EDITING,
CAP_INPAINTING,
CAP_VIDEO_GENERATION,
CAP_REASONING,
CAP_TOOL_CALL,
CAP_STRUCTURED_OUTPUT,
CAP_WEB_SEARCH,
CAP_STREAMING,
CAP_JSON_MODE,
CAP_TRANSCRIPTION,
CAP_TTS,
CAP_REALTIME,
CAP_TEXT_RENDERING,
}
)
SOURCE_ADMIN_OVERRIDE = "admin_override"
SOURCE_ENDPOINT_CONFIG = "endpoint_config"
SOURCE_PROVIDER_READER = "provider_reader"
SOURCE_COOKBOOK_HF = "cookbook_hf"
SOURCE_MODELS_DEV_REGISTRY = "models_dev_registry"
SOURCE_PROVIDER_DOCS_REGISTRY = "provider_docs_registry"
SOURCE_HEURISTIC = "heuristic"
SOURCE_CAPABILITY_PROBE = "capability_probe"
SOURCE_UNKNOWN = "unknown"
SOURCES = frozenset(
{
SOURCE_ADMIN_OVERRIDE,
SOURCE_ENDPOINT_CONFIG,
SOURCE_PROVIDER_READER,
SOURCE_COOKBOOK_HF,
SOURCE_MODELS_DEV_REGISTRY,
SOURCE_PROVIDER_DOCS_REGISTRY,
SOURCE_HEURISTIC,
SOURCE_CAPABILITY_PROBE,
SOURCE_UNKNOWN,
}
)
CONFIDENCE_EXPLICIT = "explicit"
CONFIDENCE_PROVIDER_REPORTED = "provider_reported"
CONFIDENCE_REGISTRY = "registry"
CONFIDENCE_HEURISTIC = "heuristic"
CONFIDENCE_UNKNOWN = "unknown"
CONFIDENCES = frozenset(
{
CONFIDENCE_EXPLICIT,
CONFIDENCE_PROVIDER_REPORTED,
CONFIDENCE_REGISTRY,
CONFIDENCE_HEURISTIC,
CONFIDENCE_UNKNOWN,
}
)
ASSERTION_CLAIMED = "claimed"
ASSERTION_VERIFIED = "verified"
ASSERTION_UNSUPPORTED = "unsupported"
ASSERTION_UNKNOWN = "unknown"
ASSERTION_STATUSES = frozenset(
{
ASSERTION_CLAIMED,
ASSERTION_VERIFIED,
ASSERTION_UNSUPPORTED,
ASSERTION_UNKNOWN,
}
)
PROBE_PASS = "pass"
PROBE_FAIL = "fail"
PROBE_PARTIAL = "partial"
PROBE_STATUSES = frozenset(
{
PROBE_PASS,
PROBE_FAIL,
PROBE_PARTIAL,
}
)
CONTROL_TEMPERATURE = "temperature"
CONTROL_TOP_P = "top_p"
CONTROL_TOP_K = "top_k"
CONTROL_SEED = "seed"
CONTROL_MODEL_VERSION_PIN = "model_version_pin"
CONTROL_STRICT_SCHEMA = "strict_schema"
CONTROL_TOOL_CHOICE = "tool_choice"
CONTROL_SYSTEM_PROMPT = "system_prompt"
CONTROL_PROMPT_CACHING = "prompt_caching"
CONTROL_BATCH = "batch"
CONTROL_REQUEST_HASH_CACHE = "request_hash_cache"
CONTROL_SYSTEM_FINGERPRINT = "system_fingerprint"
# Canonical reasoning control mechanisms describe how a serving path accepts
# reasoning controls. They are provider/engine evidence, not user preferences.
REASONING_CONTROL_MESSAGE_DIRECTIVE = "reasoning_message_directive" # User-message soft switch, e.g. /think or /no_think.
REASONING_CONTROL_SYSTEM_DIRECTIVE = "reasoning_system_directive" # System prompt instruction, e.g. "detailed thinking on/off".
REASONING_CONTROL_TEMPLATE_KWARG = "reasoning_template_kwarg" # Chat-template kwarg, e.g. chat_template_kwargs.enable_thinking.
REASONING_CONTROL_NATIVE_BOOL = "reasoning_native_bool" # Direct API boolean, e.g. think: true/false.
REASONING_CONTROL_STRUCTURED_OBJECT = "reasoning_structured_object" # Structured API object, e.g. thinking: {type: "..."}.
REASONING_CONTROL_BUDGET = "reasoning_budget" # Token budget control, e.g. thinkingBudget: 0/-1/N.
REASONING_CONTROL_EFFORT = "reasoning_effort" # Graded effort control, e.g. low/medium/high.
# Canonical reasoning control values describe what the provider control accepts.
# Odysseus runtime preferences can also use auto/on/off, but that is a separate
# layer that later code resolves into these provider-specific controls.
REASONING_CONTROL_VALUE_ON = "on" # Provider supports explicitly requesting reasoning on.
REASONING_CONTROL_VALUE_OFF = "off" # Provider supports explicitly requesting reasoning off.
REASONING_CONTROL_VALUE_AUTO = "auto" # Provider supports adaptive/dynamic/vendor-decided reasoning.
REASONING_CONTROL_MECHANISMS = frozenset(
{
REASONING_CONTROL_MESSAGE_DIRECTIVE,
REASONING_CONTROL_SYSTEM_DIRECTIVE,
REASONING_CONTROL_TEMPLATE_KWARG,
REASONING_CONTROL_NATIVE_BOOL,
REASONING_CONTROL_STRUCTURED_OBJECT,
REASONING_CONTROL_BUDGET,
REASONING_CONTROL_EFFORT,
}
)
REASONING_CONTROL_VALUES = frozenset(
{
REASONING_CONTROL_VALUE_ON,
REASONING_CONTROL_VALUE_OFF,
REASONING_CONTROL_VALUE_AUTO,
}
)
DETERMINISTIC_CONTROLS = frozenset(
{
CONTROL_TEMPERATURE,
CONTROL_TOP_P,
CONTROL_TOP_K,
CONTROL_SEED,
CONTROL_MODEL_VERSION_PIN,
CONTROL_STRICT_SCHEMA,
CONTROL_TOOL_CHOICE,
CONTROL_SYSTEM_PROMPT,
CONTROL_PROMPT_CACHING,
CONTROL_BATCH,
CONTROL_REQUEST_HASH_CACHE,
CONTROL_SYSTEM_FINGERPRINT,
}
)
TASK_CHAT_COMPLETIONS = "chat.completions"
TASK_EMBEDDINGS_CREATE = "embeddings.create"
TASK_IMAGE_GENERATE = "image.generate"
TASK_IMAGE_EDIT = "image.edit"
TASK_VIDEO_GENERATE = "video.generate"
TASK_AUDIO_TRANSCRIBE = "audio.transcribe"
TASK_AUDIO_SYNTHESIZE = "audio.synthesize"
TASK_RERANK = "rerank.score"
TASK_CLASSIFY = "classification.classify"
TASK_MODERATE = "moderation.moderate"
TASK_UNKNOWN = "unknown"
_FAMILY_ALIASES = {
"llm": FAMILY_CHAT,
"text": FAMILY_CHAT,
"text2text": FAMILY_CHAT,
"chat_completion": FAMILY_CHAT,
"chat_completions": FAMILY_CHAT,
"embeddings": FAMILY_EMBEDDING,
"embed": FAMILY_EMBEDDING,
"image_generation": FAMILY_IMAGE,
"image_editing": FAMILY_IMAGE,
"video_generation": FAMILY_VIDEO,
"speech": FAMILY_AUDIO,
"stt": FAMILY_AUDIO,
"tts": FAMILY_AUDIO,
"safety": FAMILY_MODERATION,
}
_MODALITY_ALIASES = {
"images": MODALITY_IMAGE,
"img": MODALITY_IMAGE,
"document": MODALITY_FILE,
"documents": MODALITY_FILE,
"files": MODALITY_FILE,
"docs": MODALITY_FILE,
"voice": MODALITY_AUDIO,
"sound": MODALITY_AUDIO,
"embeddings": MODALITY_EMBEDDING,
}
_CAPABILITY_ALIASES = {
"tools": CAP_TOOL_CALL,
"tool_calls": CAP_TOOL_CALL,
"function_calling": CAP_TOOL_CALL,
"functions": CAP_TOOL_CALL,
"image_generate": CAP_IMAGE_GENERATION,
"text_to_image": CAP_IMAGE_GENERATION,
"text-to-image": CAP_IMAGE_GENERATION,
"img2img": CAP_IMAGE_EDITING,
"image_edit": CAP_IMAGE_EDITING,
"image-editing": CAP_IMAGE_EDITING,
"text_rendering": CAP_TEXT_RENDERING,
"reasoning_effort": CAP_REASONING,
"thinking": CAP_REASONING,
"json": CAP_JSON_MODE,
"structured_outputs": CAP_STRUCTURED_OUTPUT,
"search": CAP_WEB_SEARCH,
}
_DETERMINISTIC_CONTROL_ALIASES = {
"temp": CONTROL_TEMPERATURE,
"topp": CONTROL_TOP_P,
"top-p": CONTROL_TOP_P,
"topk": CONTROL_TOP_K,
"top-k": CONTROL_TOP_K,
"version_pin": CONTROL_MODEL_VERSION_PIN,
"model_pin": CONTROL_MODEL_VERSION_PIN,
"strict_tool_schema": CONTROL_STRICT_SCHEMA,
"json_schema": CONTROL_STRICT_SCHEMA,
"tool_choice_required": CONTROL_TOOL_CHOICE,
"system": CONTROL_SYSTEM_PROMPT,
"system_message": CONTROL_SYSTEM_PROMPT,
"cache": CONTROL_REQUEST_HASH_CACHE,
"fingerprint": CONTROL_SYSTEM_FINGERPRINT,
}
_REASONING_CONTROL_ALIASES = {
"message_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE,
"user_message_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE,
"think_directive": REASONING_CONTROL_MESSAGE_DIRECTIVE,
"slash_think": REASONING_CONTROL_MESSAGE_DIRECTIVE,
"system_directive": REASONING_CONTROL_SYSTEM_DIRECTIVE,
"system_prompt_directive": REASONING_CONTROL_SYSTEM_DIRECTIVE,
"template_kwarg": REASONING_CONTROL_TEMPLATE_KWARG,
"chat_template_kwarg": REASONING_CONTROL_TEMPLATE_KWARG,
"chat_template_kwargs": REASONING_CONTROL_TEMPLATE_KWARG,
"enable_thinking": REASONING_CONTROL_TEMPLATE_KWARG,
"native_bool": REASONING_CONTROL_NATIVE_BOOL,
"think_bool": REASONING_CONTROL_NATIVE_BOOL,
"thinking_bool": REASONING_CONTROL_NATIVE_BOOL,
"structured_object": REASONING_CONTROL_STRUCTURED_OBJECT,
"reasoning_object": REASONING_CONTROL_STRUCTURED_OBJECT,
"thinking_budget": REASONING_CONTROL_BUDGET,
"budget": REASONING_CONTROL_BUDGET,
"effort": REASONING_CONTROL_EFFORT,
}
_REASONING_CONTROL_VALUE_ALIASES = {
"enabled": REASONING_CONTROL_VALUE_ON,
"enable": REASONING_CONTROL_VALUE_ON,
"true": REASONING_CONTROL_VALUE_ON,
"disabled": REASONING_CONTROL_VALUE_OFF,
"disable": REASONING_CONTROL_VALUE_OFF,
"false": REASONING_CONTROL_VALUE_OFF,
"adaptive": REASONING_CONTROL_VALUE_AUTO,
"automatic": REASONING_CONTROL_VALUE_AUTO,
"dynamic": REASONING_CONTROL_VALUE_AUTO,
"provider_auto": REASONING_CONTROL_VALUE_AUTO,
"vendor_auto": REASONING_CONTROL_VALUE_AUTO,
}
_DEFAULT_TASK_BY_FAMILY = {
FAMILY_CHAT: TASK_CHAT_COMPLETIONS,
FAMILY_EMBEDDING: TASK_EMBEDDINGS_CREATE,
FAMILY_IMAGE: TASK_IMAGE_GENERATE,
FAMILY_VIDEO: TASK_VIDEO_GENERATE,
FAMILY_AUDIO: TASK_AUDIO_TRANSCRIBE,
FAMILY_RERANK: TASK_RERANK,
FAMILY_CLASSIFICATION: TASK_CLASSIFY,
FAMILY_MODERATION: TASK_MODERATE,
FAMILY_UNKNOWN: TASK_UNKNOWN,
}
_DEFAULT_MODALITIES_BY_FAMILY = {
FAMILY_CHAT: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
FAMILY_EMBEDDING: ((MODALITY_TEXT,), (MODALITY_EMBEDDING,)),
FAMILY_IMAGE: ((MODALITY_TEXT,), (MODALITY_IMAGE,)),
FAMILY_VIDEO: ((MODALITY_TEXT,), (MODALITY_VIDEO,)),
FAMILY_AUDIO: ((MODALITY_TEXT,), (MODALITY_AUDIO,)),
FAMILY_RERANK: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
FAMILY_CLASSIFICATION: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
FAMILY_MODERATION: ((MODALITY_TEXT,), (MODALITY_TEXT,)),
FAMILY_UNKNOWN: ((), ()),
}
_DEFAULT_CAPABILITIES_BY_FAMILY = {
FAMILY_IMAGE: (CAP_IMAGE_GENERATION,),
FAMILY_VIDEO: (CAP_VIDEO_GENERATION,),
}
def _clean_token(value: Any) -> str:
return str(value or "").strip().lower().replace("-", "_").replace(" ", "_")
def _normalize_choice(value: Any, allowed: frozenset[str], aliases: Mapping[str, str], default: str) -> str:
token = _clean_token(value)
token = aliases.get(token, token)
return token if token in allowed else default
def normalize_family(value: Any) -> str:
return _normalize_choice(value, FAMILIES, _FAMILY_ALIASES, FAMILY_UNKNOWN)
def normalize_source(value: Any) -> str:
return _normalize_choice(value, SOURCES, {}, SOURCE_UNKNOWN)
def normalize_confidence(value: Any) -> str:
return _normalize_choice(value, CONFIDENCES, {}, CONFIDENCE_UNKNOWN)
def normalize_modality(value: Any) -> str:
return _normalize_choice(value, MODALITIES, _MODALITY_ALIASES, "")
def normalize_capability(value: Any) -> str:
token = _clean_token(value)
token = _CAPABILITY_ALIASES.get(token, token)
return token if token in CAPABILITIES else ""
def normalize_assertion_status(value: Any) -> str:
return _normalize_choice(value, ASSERTION_STATUSES, {}, ASSERTION_UNKNOWN)
def normalize_probe_status(value: Any) -> str:
return _normalize_choice(value, PROBE_STATUSES, {}, "")
def normalize_deterministic_control(value: Any) -> str:
token = _clean_token(value)
token = _DETERMINISTIC_CONTROL_ALIASES.get(token, token)
return token if token in DETERMINISTIC_CONTROLS else ""
def normalize_reasoning_control_mechanism(value: Any) -> str:
token = _clean_token(value)
token = _REASONING_CONTROL_ALIASES.get(token, token)
return token if token in REASONING_CONTROL_MECHANISMS else ""
def normalize_reasoning_control_value(value: Any) -> str:
token = _clean_token(value)
token = _REASONING_CONTROL_VALUE_ALIASES.get(token, token)
return token if token in REASONING_CONTROL_VALUES else ""
def _normalize_tokens(values: Any, normalizer) -> tuple[str, ...]:
if values is None:
return ()
if isinstance(values, Mapping):
values = [key for key, enabled in values.items() if enabled]
elif isinstance(values, str) or not isinstance(values, Iterable):
values = [values]
out: list[str] = []
for value in values:
token = normalizer(value)
if token and token not in out:
out.append(token)
return tuple(out)
def _normalize_limits(limits: Mapping[str, Any] | None) -> tuple[tuple[str, Any], ...]:
if not isinstance(limits, Mapping):
return ()
return tuple(sorted((str(k), v) for k, v in limits.items() if str(k).strip()))
@dataclass(frozen=True)
class Modalities:
input: tuple[str, ...] = ()
output: tuple[str, ...] = ()
@classmethod
def from_values(cls, input: Any = None, output: Any = None) -> "Modalities":
return cls(
input=_normalize_tokens(input, normalize_modality),
output=_normalize_tokens(output, normalize_modality),
)
def to_dict(self) -> dict[str, list[str]]:
return {
"input": list(self.input),
"output": list(self.output),
}
@dataclass(frozen=True)
class ModelCapability:
family: str = FAMILY_UNKNOWN
primary_task: str = TASK_UNKNOWN
modalities: Modalities = field(default_factory=Modalities)
capabilities: tuple[str, ...] = ()
limits: tuple[tuple[str, Any], ...] = ()
source: str = SOURCE_UNKNOWN
confidence: str = CONFIDENCE_UNKNOWN
@classmethod
def build(
cls,
*,
family: Any = FAMILY_UNKNOWN,
primary_task: str | None = None,
input_modalities: Any = None,
output_modalities: Any = None,
capabilities: Any = None,
limits: Mapping[str, Any] | None = None,
source: Any = SOURCE_UNKNOWN,
confidence: Any = CONFIDENCE_UNKNOWN,
) -> "ModelCapability":
normalized_family = normalize_family(family)
default_input, default_output = _DEFAULT_MODALITIES_BY_FAMILY[normalized_family]
return cls(
family=normalized_family,
primary_task=str(primary_task or _DEFAULT_TASK_BY_FAMILY[normalized_family]).strip() or TASK_UNKNOWN,
modalities=Modalities.from_values(
input_modalities if input_modalities is not None else default_input,
output_modalities if output_modalities is not None else default_output,
),
capabilities=_normalize_tokens(
capabilities if capabilities is not None else _DEFAULT_CAPABILITIES_BY_FAMILY.get(normalized_family, ()),
normalize_capability,
),
limits=_normalize_limits(limits),
source=normalize_source(source),
confidence=normalize_confidence(confidence),
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "ModelCapability":
if not isinstance(value, Mapping):
return unknown_capability()
modalities = value.get("modalities")
if not isinstance(modalities, Mapping):
modalities = {}
return cls.build(
family=value.get("family"),
primary_task=value.get("primary_task"),
input_modalities=modalities.get("input"),
output_modalities=modalities.get("output"),
capabilities=value.get("capabilities"),
limits=value.get("limits"),
source=value.get("source"),
confidence=value.get("confidence"),
)
def to_dict(self) -> dict[str, Any]:
return {
"family": self.family,
"primary_task": self.primary_task,
"modalities": self.modalities.to_dict(),
"capabilities": list(self.capabilities),
"limits": dict(self.limits),
"source": self.source,
"confidence": self.confidence,
}
@dataclass(frozen=True)
class CapabilityAssertion:
capability: str = ""
status: str = ASSERTION_UNKNOWN
source: str = SOURCE_UNKNOWN
confidence: str = CONFIDENCE_UNKNOWN
evidence: tuple[tuple[str, Any], ...] = ()
tested_at: str = ""
@classmethod
def build(
cls,
*,
capability: Any,
status: Any = ASSERTION_UNKNOWN,
source: Any = SOURCE_UNKNOWN,
confidence: Any = CONFIDENCE_UNKNOWN,
evidence: Mapping[str, Any] | None = None,
tested_at: Any = "",
) -> "CapabilityAssertion":
normalized_capability = normalize_capability(capability)
normalized_status = normalize_assertion_status(status)
if not normalized_capability:
normalized_status = ASSERTION_UNKNOWN
return cls(
capability=normalized_capability,
status=normalized_status,
source=normalize_source(source),
confidence=normalize_confidence(confidence),
evidence=_normalize_limits(evidence),
tested_at=str(tested_at or "").strip(),
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "CapabilityAssertion":
if not isinstance(value, Mapping):
return cls.build(capability="")
return cls.build(
capability=value.get("capability"),
status=value.get("status"),
source=value.get("source"),
confidence=value.get("confidence"),
evidence=value.get("evidence"),
tested_at=value.get("tested_at"),
)
def to_dict(self) -> dict[str, Any]:
return {
"capability": self.capability,
"status": self.status,
"source": self.source,
"confidence": self.confidence,
"evidence": dict(self.evidence),
"tested_at": self.tested_at,
}
@dataclass(frozen=True)
class DeterministicControl:
control: str = ""
status: str = ASSERTION_UNKNOWN
source: str = SOURCE_UNKNOWN
confidence: str = CONFIDENCE_UNKNOWN
evidence: tuple[tuple[str, Any], ...] = ()
tested_at: str = ""
@classmethod
def build(
cls,
*,
control: Any,
status: Any = ASSERTION_UNKNOWN,
source: Any = SOURCE_UNKNOWN,
confidence: Any = CONFIDENCE_UNKNOWN,
evidence: Mapping[str, Any] | None = None,
tested_at: Any = "",
) -> "DeterministicControl":
normalized_control = normalize_deterministic_control(control)
normalized_status = normalize_assertion_status(status)
if not normalized_control:
normalized_status = ASSERTION_UNKNOWN
return cls(
control=normalized_control,
status=normalized_status,
source=normalize_source(source),
confidence=normalize_confidence(confidence),
evidence=_normalize_limits(evidence),
tested_at=str(tested_at or "").strip(),
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "DeterministicControl":
if not isinstance(value, Mapping):
return cls.build(control="")
return cls.build(
control=value.get("control"),
status=value.get("status"),
source=value.get("source"),
confidence=value.get("confidence"),
evidence=value.get("evidence"),
tested_at=value.get("tested_at"),
)
def to_dict(self) -> dict[str, Any]:
return {
"control": self.control,
"status": self.status,
"source": self.source,
"confidence": self.confidence,
"evidence": dict(self.evidence),
"tested_at": self.tested_at,
}
@dataclass(frozen=True)
class CapabilityProbeResult:
provider: str
model_id: str
capability: str
status: str
tested_at: str = ""
endpoint_id: str = ""
stable_model_id: str = ""
request_hash: str = ""
response_id: str = ""
response_fingerprint: str = ""
evidence: tuple[tuple[str, Any], ...] = ()
@classmethod
def build(
cls,
*,
provider: Any,
model_id: Any,
capability: Any,
status: Any,
tested_at: Any = "",
endpoint_id: Any = "",
stable_model_id: Any = "",
request_hash: Any = "",
response_id: Any = "",
response_fingerprint: Any = "",
evidence: Mapping[str, Any] | None = None,
) -> "CapabilityProbeResult":
normalized_capability = normalize_capability(capability)
normalized_status = normalize_probe_status(status)
if not normalized_capability or not normalized_status:
normalized_status = PROBE_FAIL
return cls(
provider=str(provider or "").strip(),
model_id=str(model_id or "").strip(),
capability=normalized_capability,
status=normalized_status,
tested_at=str(tested_at or "").strip(),
endpoint_id=str(endpoint_id or "").strip(),
stable_model_id=str(stable_model_id or "").strip(),
request_hash=str(request_hash or "").strip(),
response_id=str(response_id or "").strip(),
response_fingerprint=str(response_fingerprint or "").strip(),
evidence=_normalize_limits(evidence),
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "CapabilityProbeResult":
if not isinstance(value, Mapping):
return cls.build(provider="", model_id="", capability="", status=PROBE_FAIL)
return cls.build(
provider=value.get("provider"),
endpoint_id=value.get("endpoint_id"),
model_id=value.get("model_id"),
stable_model_id=value.get("stable_model_id"),
capability=value.get("capability"),
status=value.get("status"),
tested_at=value.get("tested_at"),
request_hash=value.get("request_hash"),
response_id=value.get("response_id"),
response_fingerprint=value.get("response_fingerprint"),
evidence=value.get("evidence"),
)
def to_assertion(self) -> CapabilityAssertion:
status_map = {
PROBE_PASS: ASSERTION_VERIFIED,
PROBE_FAIL: ASSERTION_UNSUPPORTED,
PROBE_PARTIAL: ASSERTION_CLAIMED,
}
return CapabilityAssertion.build(
capability=self.capability,
status=status_map.get(self.status, ASSERTION_UNKNOWN),
source=SOURCE_CAPABILITY_PROBE,
confidence=CONFIDENCE_EXPLICIT if self.status == PROBE_PASS else CONFIDENCE_HEURISTIC,
evidence={
"provider": self.provider,
"endpoint_id": self.endpoint_id,
"model_id": self.model_id,
"stable_model_id": self.stable_model_id,
"request_hash": self.request_hash,
"response_id": self.response_id,
"response_fingerprint": self.response_fingerprint,
**dict(self.evidence),
},
tested_at=self.tested_at,
)
def to_dict(self) -> dict[str, Any]:
return {
"provider": self.provider,
"endpoint_id": self.endpoint_id,
"model_id": self.model_id,
"stable_model_id": self.stable_model_id,
"capability": self.capability,
"status": self.status,
"tested_at": self.tested_at,
"request_hash": self.request_hash,
"response_id": self.response_id,
"response_fingerprint": self.response_fingerprint,
"evidence": dict(self.evidence),
}
def capability_assertions_from_capability(
capability: ModelCapability,
*,
status: str = ASSERTION_CLAIMED,
source: str | None = None,
confidence: str | None = None,
) -> tuple[CapabilityAssertion, ...]:
return tuple(
CapabilityAssertion.build(
capability=cap,
status=status,
source=source or capability.source,
confidence=confidence or capability.confidence,
)
for cap in capability.capabilities
)
def deterministic_controls_from_values(
values: Any,
*,
status: str = ASSERTION_CLAIMED,
source: str = SOURCE_PROVIDER_READER,
confidence: str = CONFIDENCE_PROVIDER_REPORTED,
) -> tuple[DeterministicControl, ...]:
return tuple(
DeterministicControl.build(
control=control,
status=status,
source=source,
confidence=confidence,
)
for control in _normalize_tokens(values, normalize_deterministic_control)
)
@dataclass(frozen=True)
class CapabilityQuery:
surface: str
families: tuple[str, ...] = ()
primary_tasks: tuple[str, ...] = ()
input_all: tuple[str, ...] = ()
input_any: tuple[str, ...] = ()
output_all: tuple[str, ...] = ()
output_any: tuple[str, ...] = ()
modality_any: tuple[str, ...] = ()
capabilities_all: tuple[str, ...] = ()
capabilities_any: tuple[str, ...] = ()
def matches(self, capability: ModelCapability) -> bool:
input_set = set(capability.modalities.input)
output_set = set(capability.modalities.output)
modality_set = input_set | output_set
cap_set = set(capability.capabilities)
if self.families and capability.family not in self.families:
return False
if self.primary_tasks and capability.primary_task not in self.primary_tasks:
return False
if self.input_all and not set(self.input_all).issubset(input_set):
return False
if self.input_any and input_set.isdisjoint(self.input_any):
return False
if self.output_all and not set(self.output_all).issubset(output_set):
return False
if self.output_any and output_set.isdisjoint(self.output_any):
return False
if self.modality_any and modality_set.isdisjoint(self.modality_any):
return False
if self.capabilities_all and not set(self.capabilities_all).issubset(cap_set):
return False
if self.capabilities_any and cap_set.isdisjoint(self.capabilities_any):
return False
return True
DISPLAY_QUERIES = (
CapabilityQuery(
surface="chat",
families=(FAMILY_CHAT,),
input_all=(MODALITY_TEXT,),
output_all=(MODALITY_TEXT,),
),
CapabilityQuery(
surface="vision_chat",
families=(FAMILY_CHAT,),
input_all=(MODALITY_TEXT, MODALITY_IMAGE),
output_all=(MODALITY_TEXT,),
),
CapabilityQuery(
surface="document_chat",
families=(FAMILY_CHAT,),
input_all=(MODALITY_TEXT,),
input_any=(MODALITY_FILE, MODALITY_PDF),
output_all=(MODALITY_TEXT,),
),
CapabilityQuery(
surface="image_generation",
families=(FAMILY_IMAGE,),
output_all=(MODALITY_IMAGE,),
capabilities_all=(CAP_IMAGE_GENERATION,),
),
CapabilityQuery(
surface="image_editing",
families=(FAMILY_IMAGE,),
input_all=(MODALITY_IMAGE,),
output_all=(MODALITY_IMAGE,),
capabilities_any=(CAP_IMAGE_EDITING, CAP_INPAINTING),
),
CapabilityQuery(
surface="video_generation",
families=(FAMILY_VIDEO,),
output_all=(MODALITY_VIDEO,),
capabilities_all=(CAP_VIDEO_GENERATION,),
),
CapabilityQuery(
surface="audio_realtime",
families=(FAMILY_AUDIO,),
modality_any=(MODALITY_AUDIO,),
capabilities_any=(CAP_AUDIO_INPUT, CAP_AUDIO_OUTPUT, CAP_TRANSCRIPTION, CAP_TTS, CAP_REALTIME),
),
CapabilityQuery(
surface="embeddings",
families=(FAMILY_EMBEDDING,),
output_all=(MODALITY_EMBEDDING,),
),
CapabilityQuery(
surface="rerank_scoring",
families=(FAMILY_RERANK,),
),
CapabilityQuery(
surface="moderation_classification",
families=(FAMILY_MODERATION, FAMILY_CLASSIFICATION),
),
)
def display_surfaces_for(capability: ModelCapability) -> tuple[str, ...]:
return tuple(query.surface for query in DISPLAY_QUERIES if query.matches(capability))
def unknown_capability(
*,
source: str = SOURCE_UNKNOWN,
confidence: str = CONFIDENCE_UNKNOWN,
) -> ModelCapability:
return ModelCapability.build(source=source, confidence=confidence)
def capability_from_endpoint_type(model_type: Any) -> ModelCapability:
"""Return capability metadata from an explicit endpoint model type.
Missing or unknown endpoint types remain unknown here. Runtime compatibility
may still treat legacy rows as chat-capable, but this schema layer should
not turn absence of evidence into model capability truth.
"""
token = _clean_token(model_type)
if token == "llm":
return ModelCapability.build(
family=FAMILY_CHAT,
source=SOURCE_ENDPOINT_CONFIG,
confidence=CONFIDENCE_EXPLICIT,
)
if token == "image":
return ModelCapability.build(
family=FAMILY_IMAGE,
source=SOURCE_ENDPOINT_CONFIG,
confidence=CONFIDENCE_EXPLICIT,
)
return unknown_capability(source=SOURCE_ENDPOINT_CONFIG)
+95
View File
@@ -0,0 +1,95 @@
"""Vendor-specific model capability reader registry."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_ANTHROPIC,
VENDOR_GENERIC_OPENAI,
VENDOR_GOOGLE,
VENDOR_HUGGINGFACE,
VENDOR_LLAMACPP,
VENDOR_LMSTUDIO,
VENDOR_OLLAMA,
VENDOR_OPENAI,
VENDOR_OPENROUTER,
VENDOR_SGLANG,
VENDOR_UNKNOWN,
VENDOR_VLLM,
detect_vendor,
stable_model_id_for,
)
READER_MODULES = {
VENDOR_GENERIC_OPENAI: generic_openai,
VENDOR_OPENAI: openai,
VENDOR_OPENROUTER: openrouter,
VENDOR_GOOGLE: google,
VENDOR_LLAMACPP: llamacpp,
VENDOR_OLLAMA: ollama,
VENDOR_LMSTUDIO: lmstudio,
}
PLACEHOLDER_VENDOR_IDS = frozenset(
{
VENDOR_ANTHROPIC,
VENDOR_HUGGINGFACE,
VENDOR_SGLANG,
VENDOR_VLLM,
}
)
def reader_for_vendor(vendor: Any):
vendor_id = str(vendor or "").strip().lower().replace("-", "_")
return READER_MODULES.get(vendor_id, generic_openai)
def records_from_payload(
payload: Mapping[str, Any],
*,
vendor: str | None = None,
base_url: str = "",
endpoint_kind: str = "",
endpoint_id: str = "",
) -> tuple[ModelCapabilityRecord, ...]:
vendor_id = vendor or detect_vendor(base_url, endpoint_kind)
reader = reader_for_vendor(vendor_id)
if reader is generic_openai:
record_vendor = vendor_id if vendor_id not in {VENDOR_UNKNOWN, ""} else VENDOR_GENERIC_OPENAI
return reader.records_from_payload(
payload,
vendor_id=record_vendor,
endpoint_id=endpoint_id,
base_url=base_url,
)
return reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
__all__ = [
"ModelCapabilityRecord",
"PLACEHOLDER_VENDOR_IDS",
"READER_MODULES",
"VENDOR_ANTHROPIC",
"VENDOR_GENERIC_OPENAI",
"VENDOR_GOOGLE",
"VENDOR_HUGGINGFACE",
"VENDOR_LLAMACPP",
"VENDOR_LMSTUDIO",
"VENDOR_OLLAMA",
"VENDOR_OPENAI",
"VENDOR_OPENROUTER",
"VENDOR_SGLANG",
"VENDOR_UNKNOWN",
"VENDOR_VLLM",
"detect_vendor",
"reader_for_vendor",
"records_from_payload",
"stable_model_id_for",
]
+311
View File
@@ -0,0 +1,311 @@
"""Shared helpers for vendor-specific model capability readers.
Readers in this package normalize already-fetched provider payload shapes and
explicit provider fields. They do not perform network I/O and must not infer
authoritative capability from model IDs, names, display names, or ownership
labels.
"""
from __future__ import annotations
import hashlib
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any, Protocol
from urllib.parse import urlparse
from src import model_capabilities as mc
VENDOR_GENERIC_OPENAI = "generic_openai"
VENDOR_OPENAI = "openai"
VENDOR_OPENROUTER = "openrouter"
VENDOR_GOOGLE = "google"
VENDOR_ANTHROPIC = "anthropic"
VENDOR_OLLAMA = "ollama"
VENDOR_LMSTUDIO = "lmstudio"
VENDOR_LLAMACPP = "llamacpp"
VENDOR_VLLM = "vllm"
VENDOR_SGLANG = "sglang"
VENDOR_HUGGINGFACE = "huggingface"
VENDOR_UNKNOWN = "unknown"
@dataclass(frozen=True)
class ModelCapabilityRecord:
vendor: str
model_id: str
capability: mc.ModelCapability
display_name: str = ""
stable_model_id: str = ""
capability_assertions: tuple[mc.CapabilityAssertion, ...] = ()
deterministic_controls: tuple[mc.DeterministicControl, ...] = ()
raw: Mapping[str, Any] = field(default_factory=dict)
def __post_init__(self) -> None:
if not self.stable_model_id:
object.__setattr__(self, "stable_model_id", stable_model_id_for(self.vendor, self.model_id))
if not self.capability_assertions and self.capability.capabilities:
object.__setattr__(
self,
"capability_assertions",
mc.capability_assertions_from_capability(
self.capability,
status=mc.ASSERTION_CLAIMED,
source=self.capability.source,
confidence=self.capability.confidence,
),
)
def to_dict(self, *, include_raw: bool = False) -> dict[str, Any]:
data = {
"vendor": self.vendor,
"model_id": self.model_id,
"stable_model_id": self.stable_model_id,
"display_name": self.display_name,
"capability": self.capability.to_dict(),
"capability_assertions": [assertion.to_dict() for assertion in self.capability_assertions],
"deterministic_controls": [control.to_dict() for control in self.deterministic_controls],
}
if include_raw:
data["raw"] = dict(self.raw)
return data
class CapabilityReader(Protocol):
vendor: str
def records_from_payload(
self,
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
"""Normalize a provider model-list payload into capability records."""
def as_mapping(value: Any) -> Mapping[str, Any]:
return value if isinstance(value, Mapping) else {}
def as_list(value: Any) -> list[Any]:
if value is None:
return []
if isinstance(value, list):
return value
if isinstance(value, tuple):
return list(value)
return [value]
def compact_str(value: Any) -> str:
return str(value or "").strip()
def _identity_part(value: Any) -> str:
text = compact_str(value).lower()
out = []
for char in text:
out.append(char if char.isalnum() or char in {"-", "_", ".", "/", ":"} else "_")
return "".join(out).strip("_") or "unknown"
def _base_url_scope(base_url: Any) -> str:
parsed = urlparse(compact_str(base_url))
if not parsed.hostname:
return ""
port = f":{parsed.port}" if parsed.port else ""
path = parsed.path.rstrip("/")
normalized = f"{parsed.scheme or 'http'}://{parsed.hostname.lower()}{port}{path}"
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:12]
return f"url:{digest}"
def stable_model_id_for(vendor: Any, model_id: Any, *, endpoint_id: Any = "", base_url: Any = "") -> str:
vendor_part = _identity_part(vendor or VENDOR_UNKNOWN)
model_part = _identity_part(model_id)
endpoint = compact_str(endpoint_id)
if endpoint:
scope = f"endpoint:{_identity_part(endpoint)}"
else:
scope = _base_url_scope(base_url) or "global"
return f"{vendor_part}|{scope}|{model_part}"
def model_id_from(raw: Mapping[str, Any], *keys: str) -> str:
for key in keys:
value = compact_str(raw.get(key))
if value:
return value.removeprefix("models/")
return ""
def int_limit(value: Any) -> int | None:
try:
limit = int(value)
except (TypeError, ValueError):
return None
return limit if limit > 0 else None
def merge_unique(*groups: Iterable[str]) -> tuple[str, ...]:
out: list[str] = []
for group in groups:
for value in group:
token = compact_str(value)
if token and token not in out:
out.append(token)
return tuple(out)
def deterministic_controls_from_supported_parameters(values: Any) -> tuple[mc.DeterministicControl, ...]:
return mc.deterministic_controls_from_values(
values,
status=mc.ASSERTION_CLAIMED,
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
)
def openai_model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
payload = as_mapping(payload)
data = payload.get("data")
if data is None:
data = payload.get("models")
return tuple(item for item in as_list(data) if isinstance(item, Mapping))
def normalize_modality_token(value: Any) -> str:
token = compact_str(value).lower().replace("-", "_").replace(" ", "_")
aliases = {
"txt": mc.MODALITY_TEXT,
"textual": mc.MODALITY_TEXT,
"image_url": mc.MODALITY_IMAGE,
"images": mc.MODALITY_IMAGE,
"img": mc.MODALITY_IMAGE,
"audio_url": mc.MODALITY_AUDIO,
"speech": mc.MODALITY_AUDIO,
"documents": mc.MODALITY_FILE,
"document": mc.MODALITY_FILE,
"files": mc.MODALITY_FILE,
"file_search": mc.MODALITY_FILE,
"pdfs": mc.MODALITY_PDF,
"embeddings": mc.MODALITY_EMBEDDING,
}
token = aliases.get(token, token)
return mc.normalize_modality(token)
def modalities_from_value(value: Any) -> tuple[str, ...]:
if isinstance(value, str):
parts = value.replace(",", "+").replace("/", "+").split("+")
else:
parts = as_list(value)
out: list[str] = []
for part in parts:
token = normalize_modality_token(part)
if token and token not in out:
out.append(token)
return tuple(out)
def split_modality_arrow(value: Any) -> tuple[tuple[str, ...], tuple[str, ...]]:
text = compact_str(value).lower()
if not text:
return (), ()
for arrow in ("->", "=>", "to"):
if arrow in text:
left, right = text.split(arrow, 1)
return modalities_from_value(left), modalities_from_value(right)
return modalities_from_value(text), ()
def family_from_modalities(input_modalities: Iterable[str], output_modalities: Iterable[str]) -> str:
output_set = set(output_modalities)
if mc.MODALITY_EMBEDDING in output_set:
return mc.FAMILY_EMBEDDING
if mc.MODALITY_IMAGE in output_set:
return mc.FAMILY_IMAGE
if mc.MODALITY_VIDEO in output_set:
return mc.FAMILY_VIDEO
if mc.MODALITY_AUDIO in output_set:
return mc.FAMILY_AUDIO
if mc.MODALITY_TEXT in output_set:
return mc.FAMILY_CHAT
return mc.FAMILY_UNKNOWN
def primary_task_for_family(family: str, capabilities: Iterable[str] = ()) -> str | None:
caps = set(capabilities)
if family == mc.FAMILY_IMAGE and (mc.CAP_IMAGE_EDITING in caps or mc.CAP_INPAINTING in caps):
return mc.TASK_IMAGE_EDIT
if family == mc.FAMILY_AUDIO and mc.CAP_TTS in caps:
return mc.TASK_AUDIO_SYNTHESIZE
if family == mc.FAMILY_AUDIO and mc.CAP_TRANSCRIPTION in caps:
return mc.TASK_AUDIO_TRANSCRIBE
return None
def build_capability(
*,
family: str,
input_modalities: Iterable[str] = (),
output_modalities: Iterable[str] = (),
capabilities: Iterable[str] = (),
limits: Mapping[str, Any] | None = None,
confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED,
) -> mc.ModelCapability:
return mc.ModelCapability.build(
family=family,
primary_task=primary_task_for_family(family, capabilities),
input_modalities=tuple(input_modalities),
output_modalities=tuple(output_modalities),
capabilities=tuple(capabilities),
limits=limits,
source=mc.SOURCE_PROVIDER_READER,
confidence=confidence,
)
def detect_vendor(base_url: Any = "", endpoint_kind: Any = "") -> str:
kind = compact_str(endpoint_kind).lower().replace("-", "_")
kind_map = {
"openai": VENDOR_OPENAI,
"openrouter": VENDOR_OPENROUTER,
"google": VENDOR_GOOGLE,
"gemini": VENDOR_GOOGLE,
"anthropic": VENDOR_ANTHROPIC,
"ollama": VENDOR_OLLAMA,
"lmstudio": VENDOR_LMSTUDIO,
"lm_studio": VENDOR_LMSTUDIO,
"llamacpp": VENDOR_LLAMACPP,
"llama_cpp": VENDOR_LLAMACPP,
"vllm": VENDOR_VLLM,
"sglang": VENDOR_SGLANG,
"huggingface": VENDOR_HUGGINGFACE,
"hf": VENDOR_HUGGINGFACE,
}
if kind in kind_map:
return kind_map[kind]
parsed = urlparse(compact_str(base_url))
host = (parsed.hostname or "").lower()
port = parsed.port
if host.endswith("openrouter.ai"):
return VENDOR_OPENROUTER
if host.endswith("openai.com"):
return VENDOR_OPENAI
if host.endswith("anthropic.com"):
return VENDOR_ANTHROPIC
if host.endswith("googleapis.com"):
return VENDOR_GOOGLE
if host.endswith("ollama.com") or port == 11434:
return VENDOR_OLLAMA
if port == 1234:
return VENDOR_LMSTUDIO
if port == 8000:
return VENDOR_VLLM
if port == 30000:
return VENDOR_SGLANG
return VENDOR_GENERIC_OPENAI if host else VENDOR_UNKNOWN
@@ -0,0 +1,58 @@
"""Reader for bare OpenAI-compatible model-list payloads."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_GENERIC_OPENAI,
compact_str,
model_id_from,
openai_model_items,
stable_model_id_for,
)
vendor = VENDOR_GENERIC_OPENAI
def record_from_model(
raw: Mapping[str, Any],
*,
vendor_id: str = VENDOR_GENERIC_OPENAI,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = model_id_from(raw, "id", "name", "model")
if not model_id:
return None
capability = mc.unknown_capability(
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_UNKNOWN,
)
return ModelCapabilityRecord(
vendor=vendor_id,
model_id=model_id,
stable_model_id=stable_model_id_for(vendor_id, model_id, endpoint_id=endpoint_id, base_url=base_url),
display_name=compact_str(raw.get("display_name") or raw.get("name")),
capability=capability,
raw=raw,
)
def records_from_payload(
payload: Mapping[str, Any],
*,
vendor_id: str = VENDOR_GENERIC_OPENAI,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
records: list[ModelCapabilityRecord] = []
for item in openai_model_items(payload):
record = record_from_model(item, vendor_id=vendor_id, endpoint_id=endpoint_id, base_url=base_url)
if record:
records.append(record)
return tuple(records)
+60
View File
@@ -0,0 +1,60 @@
"""Google Gemini model metadata reader."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src.model_capability_readers import google_ai_studio_mapping as ai_studio
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_GOOGLE,
as_list,
compact_str,
stable_model_id_for,
)
vendor = VENDOR_GOOGLE
def _model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
models = payload.get("models") if isinstance(payload, Mapping) else None
if models is None and isinstance(payload, Mapping) and payload.get("name"):
models = [payload]
return tuple(item for item in as_list(models) if isinstance(item, Mapping))
def record_from_model(
raw: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = ai_studio.google_model_id(raw)
if not model_id:
return None
return ModelCapabilityRecord(
vendor=VENDOR_GOOGLE,
model_id=model_id,
stable_model_id=stable_model_id_for(VENDOR_GOOGLE, model_id, endpoint_id=endpoint_id, base_url=base_url),
display_name=compact_str(raw.get("displayName")) or model_id,
capability=ai_studio.capability_from_model(raw),
deterministic_controls=ai_studio.deterministic_controls_from_model(raw),
raw=raw,
)
def records_from_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
records: list[ModelCapabilityRecord] = []
for item in _model_items(payload):
record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)
if record:
records.append(record)
return tuple(records)
@@ -0,0 +1,162 @@
"""Google AI Studio / Gemini native Models API capability mapping.
This module maps already-fetched `models.list` and `models.get` payloads into
Odysseus' canonical model capability shape. It performs no network I/O and
does not infer model capabilities from model IDs, display names, or product
families. Only fields explicitly returned by Google's Model resource are
mapped here.
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers.base import as_list, compact_str, int_limit
METHOD_GENERATE_CONTENT = "generateContent"
METHOD_GENERATE_MESSAGE = "generateMessage"
METHOD_GENERATE_TEXT = "generateText"
METHOD_GENERATE_ANSWER = "generateAnswer"
METHOD_EMBED_CONTENT = "embedContent"
METHOD_ASYNC_BATCH_EMBED = "asyncBatchEmbedContent"
METHOD_PREDICT = "predict"
METHOD_PREDICT_LONG_RUNNING = "predictLongRunning"
METHOD_BATCH_GENERATE = "batchGenerateContent"
METHOD_CREATE_CACHED_CONTENT = "createCachedContent"
TEXT_GENERATION_METHODS = frozenset(
{
METHOD_GENERATE_CONTENT,
METHOD_GENERATE_MESSAGE,
METHOD_GENERATE_TEXT,
METHOD_GENERATE_ANSWER,
}
)
EMBEDDING_METHODS = frozenset({METHOD_EMBED_CONTENT, METHOD_ASYNC_BATCH_EMBED})
BATCH_METHODS = frozenset({METHOD_BATCH_GENERATE, METHOD_ASYNC_BATCH_EMBED})
MODEL_FIELD_MAP = {
"name": "vendor resource name",
"baseModelId": "vendor model id",
"displayName": "display name",
"description": "display description only",
"inputTokenLimit": "limits.input_tokens and limits.context_tokens",
"outputTokenLimit": "limits.output_tokens",
"supportedGenerationMethods": "provider method support signal",
"thinking": "capabilities.reasoning when true",
"temperature": "deterministic_controls.temperature when present",
"maxTemperature": "deterministic_controls.temperature when present",
"topP": "deterministic_controls.top_p when present",
"topK": "deterministic_controls.top_k when present",
}
def google_model_id(raw: Mapping[str, Any]) -> str:
value = compact_str(raw.get("baseModelId")) or compact_str(raw.get("name"))
return value.removeprefix("models/")
def supported_methods(raw: Mapping[str, Any]) -> frozenset[str]:
return frozenset(compact_str(method) for method in as_list(raw.get("supportedGenerationMethods")) if method)
def limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]:
limits: dict[str, Any] = {}
input_limit = int_limit(raw.get("inputTokenLimit"))
output_limit = int_limit(raw.get("outputTokenLimit"))
if input_limit:
limits["input_tokens"] = input_limit
limits["context_tokens"] = input_limit
if output_limit:
limits["output_tokens"] = output_limit
return limits
def _capability(
*,
family: str,
input_modalities: tuple[str, ...],
output_modalities: tuple[str, ...],
capabilities: tuple[str, ...] = (),
limits: Mapping[str, Any] | None = None,
primary_task: str | None = None,
source: str = mc.SOURCE_PROVIDER_READER,
confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED,
) -> mc.ModelCapability:
return mc.ModelCapability.build(
family=family,
primary_task=primary_task,
input_modalities=input_modalities,
output_modalities=output_modalities,
capabilities=capabilities,
limits=limits,
source=source,
confidence=confidence,
)
def capability_from_model(raw: Mapping[str, Any]) -> mc.ModelCapability:
methods = supported_methods(raw)
capabilities: list[str] = []
if raw.get("thinking") is True:
capabilities.append(mc.CAP_REASONING)
if methods & EMBEDDING_METHODS and not methods & TEXT_GENERATION_METHODS:
return _capability(
family=mc.FAMILY_EMBEDDING,
input_modalities=(mc.MODALITY_TEXT,),
output_modalities=(mc.MODALITY_EMBEDDING,),
capabilities=tuple(capabilities),
limits=limits_from_model(raw),
)
# `generateContent` proves the model supports Google's content generation
# method, but the Model resource does not expose input/output modalities.
# Keep the model unknown instead of guessing chat/image/audio/video from ID.
if methods & TEXT_GENERATION_METHODS:
return _capability(
family=mc.FAMILY_UNKNOWN,
input_modalities=(),
output_modalities=(),
capabilities=tuple(capabilities),
limits=limits_from_model(raw),
)
capability = mc.unknown_capability(
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_UNKNOWN,
)
limits = limits_from_model(raw)
if limits or capabilities:
return _capability(
family=mc.FAMILY_UNKNOWN,
input_modalities=(),
output_modalities=(),
capabilities=tuple(capabilities),
limits=limits,
)
return capability
def deterministic_controls_from_model(raw: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]:
methods = supported_methods(raw)
controls: list[str] = []
if "temperature" in raw or "maxTemperature" in raw:
controls.append(mc.CONTROL_TEMPERATURE)
if "topP" in raw:
controls.append(mc.CONTROL_TOP_P)
if raw.get("topK") not in (None, ""):
controls.append(mc.CONTROL_TOP_K)
if METHOD_CREATE_CACHED_CONTENT in methods:
controls.append(mc.CONTROL_PROMPT_CACHING)
if methods & BATCH_METHODS:
controls.append(mc.CONTROL_BATCH)
return mc.deterministic_controls_from_values(
controls,
status=mc.ASSERTION_CLAIMED,
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
)
+428
View File
@@ -0,0 +1,428 @@
"""llama.cpp server capability reader.
llama-server exposes OpenAI-compatible model IDs through /v1/models, but its
useful runtime metadata lives in native endpoints such as /props and /slots.
This reader can normalize each payload independently and can merge the three
payloads when the probe script has them all.
"""
from __future__ import annotations
from collections.abc import Mapping
from pathlib import PurePosixPath
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers import generic_openai
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_LLAMACPP,
as_list,
as_mapping,
build_capability,
compact_str,
deterministic_controls_from_supported_parameters,
int_limit,
merge_unique,
model_id_from,
openai_model_items,
stable_model_id_for,
)
vendor = VENDOR_LLAMACPP
_SAMPLER_CONTROL_MAP = {
"temperature": mc.CONTROL_TEMPERATURE,
"top_p": mc.CONTROL_TOP_P,
}
def _model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
payload = as_mapping(payload)
data_items = openai_model_items(payload)
if data_items:
return data_items
return tuple(item for item in as_list(payload.get("models")) if isinstance(item, Mapping))
def _server_model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]:
return tuple(item for item in as_list(as_mapping(payload).get("models")) if isinstance(item, Mapping))
def _model_id_from_props(payload: Mapping[str, Any]) -> str:
payload = as_mapping(payload)
model_alias = compact_str(payload.get("model_alias"))
if model_alias:
return model_alias
model_path = compact_str(payload.get("model_path"))
if model_path:
return PurePosixPath(model_path).name
return ""
def _capability_tokens_from_server_model(raw: Mapping[str, Any]) -> tuple[str, ...]:
out: list[str] = []
for value in as_list(raw.get("capabilities")):
token = compact_str(value).lower().replace("-", "_")
if token in {"embedding", "embeddings"}:
continue
if token in {"rerank", "reranking"}:
continue
if token in {"completion", "completions", "chat"}:
continue
cap = mc.normalize_capability(token)
if cap and cap not in out:
out.append(cap)
return tuple(out)
def _family_from_server_model(raw: Mapping[str, Any]) -> str:
capabilities = {compact_str(value).lower().replace("-", "_") for value in as_list(raw.get("capabilities"))}
if "embedding" in capabilities or "embeddings" in capabilities:
return mc.FAMILY_EMBEDDING
if "rerank" in capabilities or "reranking" in capabilities:
return mc.FAMILY_RERANK
if "completion" in capabilities or "completions" in capabilities or "chat" in capabilities:
return mc.FAMILY_CHAT
return mc.FAMILY_UNKNOWN
def _matching_server_model(payload: Mapping[str, Any], model_id: str) -> Mapping[str, Any]:
for item in _server_model_entries(payload):
if model_id in {
model_id_from(item, "id", "name", "model"),
compact_str(item.get("name")),
compact_str(item.get("model")),
}:
return item
return {}
def _limits_from_model_entry(raw: Mapping[str, Any]) -> dict[str, Any]:
meta = as_mapping(raw.get("meta"))
limits: dict[str, Any] = {}
n_ctx_train = int_limit(raw.get("n_ctx_train") or meta.get("n_ctx_train"))
n_params = int_limit(raw.get("n_params") or meta.get("n_params"))
size = int_limit(raw.get("size") or meta.get("size"))
if n_ctx_train:
limits["training_context_tokens"] = n_ctx_train
if n_params:
limits["parameters"] = n_params
if size:
limits["model_bytes"] = size
return limits
def _props_params(payload: Mapping[str, Any]) -> Mapping[str, Any]:
return as_mapping(as_mapping(payload.get("default_generation_settings")).get("params"))
def _limits_from_props(payload: Mapping[str, Any], slots_payload: Any = None) -> dict[str, Any]:
default_settings = as_mapping(payload.get("default_generation_settings"))
limits: dict[str, Any] = {}
n_ctx = int_limit(default_settings.get("n_ctx"))
total_slots = int_limit(payload.get("total_slots"))
if not n_ctx and isinstance(slots_payload, list):
slot_contexts = [int_limit(as_mapping(slot).get("n_ctx")) for slot in slots_payload]
slot_contexts = [value for value in slot_contexts if value]
if slot_contexts:
n_ctx = min(slot_contexts)
if n_ctx:
limits["context_tokens"] = n_ctx
if total_slots:
limits["parallel_slots"] = total_slots
elif isinstance(slots_payload, list) and slots_payload:
limits["parallel_slots"] = len(slots_payload)
return limits
def _modalities_from_props(payload: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]:
modalities = as_mapping(payload.get("modalities"))
input_modalities = [mc.MODALITY_TEXT]
output_modalities = [mc.MODALITY_TEXT]
if modalities.get("vision") is True:
input_modalities.append(mc.MODALITY_IMAGE)
if modalities.get("audio") is True:
input_modalities.append(mc.MODALITY_AUDIO)
return tuple(input_modalities), tuple(output_modalities)
def _capabilities_from_props(payload: Mapping[str, Any]) -> tuple[str, ...]:
caps = as_mapping(payload.get("chat_template_caps"))
params = _props_params(payload)
out: list[str] = []
if caps.get("supports_tools") is True or caps.get("supports_tool_calls") is True:
out.append(mc.CAP_TOOL_CALL)
if params.get("stream") is not None:
out.append(mc.CAP_STREAMING)
if as_mapping(payload.get("modalities")).get("vision") is True:
out.append(mc.CAP_VISION)
if as_mapping(payload.get("modalities")).get("audio") is True:
out.append(mc.CAP_AUDIO_INPUT)
return tuple(out)
def _unsupported_assertions_from_props(payload: Mapping[str, Any]) -> tuple[mc.CapabilityAssertion, ...]:
modalities = as_mapping(payload.get("modalities"))
assertions: list[mc.CapabilityAssertion] = []
if modalities.get("vision") is False:
assertions.append(
mc.CapabilityAssertion.build(
capability=mc.CAP_VISION,
status=mc.ASSERTION_UNSUPPORTED,
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
evidence={"field": "modalities.vision"},
)
)
if modalities.get("audio") is False:
assertions.append(
mc.CapabilityAssertion.build(
capability=mc.CAP_AUDIO_INPUT,
status=mc.ASSERTION_UNSUPPORTED,
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_PROVIDER_REPORTED,
evidence={"field": "modalities.audio"},
)
)
return tuple(assertions)
def _deterministic_controls_from_props(payload: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]:
controls: list[str] = []
params = _props_params(payload)
for key in ("temperature", "top_p", "seed"):
if key in params:
controls.append(key)
for sampler in as_list(params.get("samplers")):
control = _SAMPLER_CONTROL_MAP.get(compact_str(sampler).lower())
if control:
controls.append(control)
template_caps = as_mapping(payload.get("chat_template_caps"))
if template_caps.get("supports_system_role") is True:
controls.append(mc.CONTROL_SYSTEM_PROMPT)
if template_caps.get("supports_tools") is True or template_caps.get("supports_tool_calls") is True:
controls.append(mc.CONTROL_TOOL_CHOICE)
return deterministic_controls_from_supported_parameters(merge_unique(controls))
def _capability_for_family(
family: str,
*,
capabilities: tuple[str, ...] = (),
limits: Mapping[str, Any] | None = None,
props_payload: Mapping[str, Any] | None = None,
) -> mc.ModelCapability:
if family == mc.FAMILY_EMBEDDING:
return build_capability(
family=mc.FAMILY_EMBEDDING,
input_modalities=(mc.MODALITY_TEXT,),
output_modalities=(mc.MODALITY_EMBEDDING,),
capabilities=capabilities,
limits=limits,
)
if family == mc.FAMILY_RERANK:
return build_capability(
family=mc.FAMILY_RERANK,
input_modalities=(mc.MODALITY_TEXT,),
output_modalities=(mc.MODALITY_TEXT,),
capabilities=capabilities,
limits=limits,
)
if props_payload:
input_modalities, output_modalities = _modalities_from_props(props_payload)
else:
input_modalities, output_modalities = (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,)
return build_capability(
family=mc.FAMILY_CHAT,
input_modalities=input_modalities,
output_modalities=output_modalities,
capabilities=capabilities,
limits=limits,
)
def _record(
*,
model_id: str,
family: str,
capabilities: tuple[str, ...] = (),
limits: Mapping[str, Any] | None = None,
props_payload: Mapping[str, Any] | None = None,
deterministic_controls: tuple[mc.DeterministicControl, ...] = (),
extra_assertions: tuple[mc.CapabilityAssertion, ...] = (),
raw: Mapping[str, Any] | None = None,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord:
capability = _capability_for_family(
family,
capabilities=capabilities,
limits=limits,
props_payload=props_payload,
)
return ModelCapabilityRecord(
vendor=VENDOR_LLAMACPP,
model_id=model_id,
stable_model_id=stable_model_id_for(VENDOR_LLAMACPP, model_id, endpoint_id=endpoint_id, base_url=base_url),
display_name=model_id,
capability=capability,
capability_assertions=(
mc.capability_assertions_from_capability(
capability,
status=mc.ASSERTION_CLAIMED,
source=capability.source,
confidence=capability.confidence,
)
+ extra_assertions
),
deterministic_controls=deterministic_controls,
raw=raw or {},
)
def record_from_model_payload(
raw: Mapping[str, Any],
*,
server_model: Mapping[str, Any] | None = None,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = model_id_from(raw, "id", "name", "model")
if not model_id:
return None
server_model = as_mapping(server_model)
family = _family_from_server_model(server_model) if server_model else mc.FAMILY_UNKNOWN
if family == mc.FAMILY_UNKNOWN:
return generic_openai.record_from_model(
raw,
vendor_id=VENDOR_LLAMACPP,
endpoint_id=endpoint_id,
base_url=base_url,
)
capabilities = _capability_tokens_from_server_model(server_model)
return _record(
model_id=model_id,
family=family,
capabilities=capabilities,
limits=_limits_from_model_entry(raw),
raw=raw,
endpoint_id=endpoint_id,
base_url=base_url,
)
def record_from_props_payload(
payload: Mapping[str, Any],
*,
slots_payload: Any = None,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
payload = as_mapping(payload)
model_id = _model_id_from_props(payload)
if not model_id:
return None
return _record(
model_id=model_id,
family=mc.FAMILY_CHAT,
capabilities=_capabilities_from_props(payload),
limits=_limits_from_props(payload, slots_payload),
props_payload=payload,
deterministic_controls=_deterministic_controls_from_props(payload),
extra_assertions=_unsupported_assertions_from_props(payload),
raw=payload,
endpoint_id=endpoint_id,
base_url=base_url,
)
def records_from_payloads(
*,
models_payload: Mapping[str, Any] | None = None,
props_payload: Mapping[str, Any] | None = None,
slots_payload: Any = None,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
props_payload = as_mapping(props_payload)
models_payload = as_mapping(models_payload)
props_record = (
record_from_props_payload(props_payload, slots_payload=slots_payload, endpoint_id=endpoint_id, base_url=base_url)
if props_payload
else None
)
if not models_payload:
return (props_record,) if props_record else ()
records: list[ModelCapabilityRecord] = []
for item in _model_entries(models_payload):
model_id = model_id_from(item, "id", "name", "model")
if not model_id:
continue
server_model = _matching_server_model(models_payload, model_id)
model_record = record_from_model_payload(
item,
server_model=server_model,
endpoint_id=endpoint_id,
base_url=base_url,
)
if not model_record:
continue
if props_record and props_record.model_id == model_id:
limits = {**dict(model_record.capability.limits), **dict(props_record.capability.limits)}
capability = _capability_for_family(
props_record.capability.family,
capabilities=merge_unique(model_record.capability.capabilities, props_record.capability.capabilities),
limits=limits,
props_payload=props_payload,
)
records.append(
ModelCapabilityRecord(
vendor=VENDOR_LLAMACPP,
model_id=model_id,
stable_model_id=stable_model_id_for(
VENDOR_LLAMACPP,
model_id,
endpoint_id=endpoint_id,
base_url=base_url,
),
display_name=model_id,
capability=capability,
capability_assertions=(
mc.capability_assertions_from_capability(
capability,
status=mc.ASSERTION_CLAIMED,
source=capability.source,
confidence=capability.confidence,
)
+ _unsupported_assertions_from_props(props_payload)
),
deterministic_controls=props_record.deterministic_controls,
raw={"models": item, "props": props_payload, "slots": slots_payload or []},
)
)
else:
records.append(model_record)
if not records and props_record:
records.append(props_record)
return tuple(records)
def records_from_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
payload = as_mapping(payload)
if not payload:
return ()
if "default_generation_settings" in payload or "chat_template_caps" in payload:
record = record_from_props_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
return (record,) if record else ()
if "models" in payload or "data" in payload:
return records_from_payloads(models_payload=payload, endpoint_id=endpoint_id, base_url=base_url)
return ()
+186
View File
@@ -0,0 +1,186 @@
"""LM Studio native model metadata reader."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers import generic_openai
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_LMSTUDIO,
as_list,
as_mapping,
build_capability,
compact_str,
int_limit,
merge_unique,
model_id_from,
openai_model_items,
stable_model_id_for,
)
vendor = VENDOR_LMSTUDIO
def _loaded_instance_contexts(raw: Mapping[str, Any]) -> tuple[int, ...]:
contexts: list[int] = []
for instance in as_list(raw.get("loaded_instances")):
instance_payload = as_mapping(instance)
config = as_mapping(instance_payload.get("config"))
value = int_limit(instance_payload.get("context_length")) or int_limit(
config.get("context_length")
)
if value:
contexts.append(value)
return tuple(contexts)
def _limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]:
limits: dict[str, Any] = {}
loaded_contexts = _loaded_instance_contexts(raw)
loaded_context = int_limit(raw.get("loaded_context_length")) or (
min(loaded_contexts) if loaded_contexts else None
)
configured_context = int_limit(raw.get("context_length")) or int_limit(raw.get("contextLength"))
max_context = int_limit(raw.get("max_context_length")) or int_limit(raw.get("maxContextLength"))
context_tokens = loaded_context or configured_context or max_context
if context_tokens:
limits["context_tokens"] = context_tokens
if max_context and max_context != context_tokens:
limits["max_context_tokens"] = max_context
return limits
def _family_from_type(raw: Mapping[str, Any]) -> str:
kind = compact_str(raw.get("type") or raw.get("model_type") or raw.get("task")).lower().replace("-", "_")
if kind in {"embedding", "embeddings", "text_embedding", "text_embeddings"}:
return mc.FAMILY_EMBEDDING
if kind in {"llm", "chat", "vlm", "vision", "text_generation"}:
return mc.FAMILY_CHAT
return mc.FAMILY_UNKNOWN
def _capabilities_from_native_payload(raw: Mapping[str, Any]) -> tuple[str, ...]:
capabilities_payload = as_mapping(raw.get("capabilities"))
capabilities: list[str] = []
if capabilities_payload.get("vision") is True:
capabilities.append(mc.CAP_VISION)
if (
capabilities_payload.get("trained_for_tool_use") is True
or capabilities_payload.get("tools") is True
or capabilities_payload.get("tool_use") is True
):
capabilities.append(mc.CAP_TOOL_CALL)
if capabilities_payload.get("reasoning"):
capabilities.append(mc.CAP_REASONING)
return merge_unique(capabilities)
def _unknown_record(
raw: Mapping[str, Any],
model_id: str,
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord:
return ModelCapabilityRecord(
vendor=VENDOR_LMSTUDIO,
model_id=model_id,
stable_model_id=stable_model_id_for(
VENDOR_LMSTUDIO,
model_id,
endpoint_id=endpoint_id,
base_url=base_url,
),
display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id,
capability=mc.unknown_capability(
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_UNKNOWN,
),
raw=raw,
)
def record_from_native_model(
raw: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = model_id_from(raw, "key", "id", "model", "name")
if not model_id:
return None
family = _family_from_type(raw)
capabilities = _capabilities_from_native_payload(raw)
if family == mc.FAMILY_UNKNOWN and capabilities:
family = mc.FAMILY_CHAT
if family == mc.FAMILY_EMBEDDING:
input_modalities = (mc.MODALITY_TEXT,)
output_modalities = (mc.MODALITY_EMBEDDING,)
elif family == mc.FAMILY_CHAT and mc.CAP_VISION in capabilities:
input_modalities = (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
output_modalities = (mc.MODALITY_TEXT,)
elif family == mc.FAMILY_CHAT:
input_modalities = (mc.MODALITY_TEXT,)
output_modalities = (mc.MODALITY_TEXT,)
else:
return generic_openai.record_from_model(
raw,
vendor_id=VENDOR_LMSTUDIO,
endpoint_id=endpoint_id,
base_url=base_url,
) or _unknown_record(
raw,
model_id,
endpoint_id=endpoint_id,
base_url=base_url,
)
capability = build_capability(
family=family,
input_modalities=input_modalities,
output_modalities=output_modalities,
capabilities=capabilities,
limits=_limits_from_model(raw),
)
return ModelCapabilityRecord(
vendor=VENDOR_LMSTUDIO,
model_id=model_id,
stable_model_id=stable_model_id_for(
VENDOR_LMSTUDIO,
model_id,
endpoint_id=endpoint_id,
base_url=base_url,
),
display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id,
capability=capability,
raw=raw,
)
def records_from_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
records: list[ModelCapabilityRecord] = []
for item in openai_model_items(payload):
record = record_from_native_model(item, endpoint_id=endpoint_id, base_url=base_url)
if record:
records.append(record)
if records:
return tuple(records)
for item in as_list(as_mapping(payload).get("models")):
if not isinstance(item, Mapping):
continue
record = record_from_native_model(item, endpoint_id=endpoint_id, base_url=base_url)
if record:
records.append(record)
return tuple(records)
+204
View File
@@ -0,0 +1,204 @@
"""Ollama native API capability reader."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_OLLAMA,
as_list,
as_mapping,
build_capability,
compact_str,
int_limit,
merge_unique,
model_id_from,
stable_model_id_for,
)
vendor = VENDOR_OLLAMA
_CAPABILITY_MAP = {
"completion": None,
"completions": None,
"chat": None,
"thinking": mc.CAP_REASONING,
"reasoning": mc.CAP_REASONING,
"vision": mc.CAP_VISION,
"tools": mc.CAP_TOOL_CALL,
"tool": mc.CAP_TOOL_CALL,
"embedding": None,
"embeddings": None,
}
def _capability_tokens(values: Any) -> tuple[str, ...]:
out: list[str] = []
for value in as_list(values):
token = compact_str(value).lower().replace("-", "_")
cap = _CAPABILITY_MAP.get(token)
if cap and cap not in out:
out.append(cap)
return tuple(out)
def _family_from_ollama_capabilities(values: Any) -> str:
tokens = {compact_str(value).lower().replace("-", "_") for value in as_list(values)}
if tokens and tokens.issubset({"embedding", "embeddings"}):
return mc.FAMILY_EMBEDDING
if "embedding" in tokens or "embeddings" in tokens:
return mc.FAMILY_EMBEDDING
if tokens.intersection({"completion", "completions", "chat", "thinking", "reasoning", "tools", "tool", "vision"}):
return mc.FAMILY_CHAT
return mc.FAMILY_UNKNOWN
def _parameters_mapping(value: Any) -> Mapping[str, Any]:
if isinstance(value, Mapping):
return value
text = compact_str(value)
if not text:
return {}
parsed: dict[str, str] = {}
for line in text.splitlines():
parts = line.strip().split(None, 1)
if len(parts) == 2:
parsed[parts[0]] = parts[1]
return parsed
def _modalities_for_family(family: str, capabilities: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]:
if family == mc.FAMILY_EMBEDDING:
return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,)
if family == mc.FAMILY_CHAT and mc.CAP_VISION in capabilities:
return (mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)
if family == mc.FAMILY_CHAT:
return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,)
return (), ()
def _first_int_by_key_shape(*mappings: Mapping[str, Any], exact_keys: tuple[str, ...] = ()) -> int | None:
for key in exact_keys:
for mapping in mappings:
value = int_limit(mapping.get(key))
if value:
return value
for mapping in mappings:
for key, value in mapping.items():
key_text = compact_str(key).lower()
if key_text == "context_length" or key_text.endswith(".context_length"):
limit = int_limit(value)
if limit:
return limit
return None
def _limits_from_show(raw: Mapping[str, Any]) -> dict[str, Any]:
model_info = as_mapping(raw.get("model_info"))
parameters = _parameters_mapping(raw.get("parameters"))
details = as_mapping(raw.get("details"))
limits: dict[str, Any] = {}
context_tokens = _first_int_by_key_shape(
raw,
model_info,
parameters,
details,
exact_keys=("context_length", "num_ctx"),
)
if context_tokens:
limits["context_tokens"] = context_tokens
return limits
def record_from_show_payload(
model_id: str,
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = compact_str(model_id) or model_id_from(payload, "model", "name")
if not model_id:
return None
capability_values = payload.get("capabilities")
capabilities = _capability_tokens(capability_values)
family = _family_from_ollama_capabilities(capability_values)
if family == mc.FAMILY_UNKNOWN:
capability = mc.unknown_capability(
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_UNKNOWN,
)
else:
input_modalities, output_modalities = _modalities_for_family(family, capabilities)
capability = build_capability(
family=family,
input_modalities=input_modalities,
output_modalities=output_modalities,
capabilities=merge_unique(capabilities),
limits=_limits_from_show(payload),
)
return ModelCapabilityRecord(
vendor=VENDOR_OLLAMA,
model_id=model_id,
stable_model_id=stable_model_id_for(VENDOR_OLLAMA, model_id, endpoint_id=endpoint_id, base_url=base_url),
display_name=model_id,
capability=capability,
raw=payload,
)
def records_from_tags_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
records: list[ModelCapabilityRecord] = []
for item in as_list(as_mapping(payload).get("models")):
if not isinstance(item, Mapping):
continue
model_id = model_id_from(item, "model", "name")
if not model_id:
continue
records.append(
ModelCapabilityRecord(
vendor=VENDOR_OLLAMA,
model_id=model_id,
stable_model_id=stable_model_id_for(
VENDOR_OLLAMA,
model_id,
endpoint_id=endpoint_id,
base_url=base_url,
),
display_name=model_id,
capability=mc.unknown_capability(
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_UNKNOWN,
),
raw=item,
)
)
return tuple(records)
def records_from_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
payload = as_mapping(payload)
if "models" in payload:
return records_from_tags_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
record = record_from_show_payload(
model_id_from(payload, "model", "name"),
payload,
endpoint_id=endpoint_id,
base_url=base_url,
)
return (record,) if record else ()
+65
View File
@@ -0,0 +1,65 @@
"""OpenAI Models API capability reader.
OpenAI's `/v1/models` list/retrieve shape currently provides model identity
metadata only: `id`, `object`, `created`, and `owned_by`. Those fields prove
availability, not model capabilities, so this reader keeps capabilities
unknown unless OpenAI adds explicit capability fields to the API shape later.
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_OPENAI,
compact_str,
model_id_from,
openai_model_items,
stable_model_id_for,
)
vendor = VENDOR_OPENAI
OFFICIAL_MODEL_FIELDS = frozenset({"id", "object", "created", "owned_by"})
def record_from_model(
raw: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = model_id_from(raw, "id")
if not model_id:
return None
return ModelCapabilityRecord(
vendor=VENDOR_OPENAI,
model_id=model_id,
stable_model_id=stable_model_id_for(VENDOR_OPENAI, model_id, endpoint_id=endpoint_id, base_url=base_url),
display_name=compact_str(raw.get("name") or raw.get("display_name")),
capability=mc.unknown_capability(
source=mc.SOURCE_PROVIDER_READER,
confidence=mc.CONFIDENCE_UNKNOWN,
),
raw=raw,
)
def records_from_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
records: list[ModelCapabilityRecord] = []
for item in openai_model_items(payload):
record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)
if record:
records.append(record)
return tuple(records)
+200
View File
@@ -0,0 +1,200 @@
"""OpenRouter model catalog capability reader."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from src import model_capabilities as mc
from src.model_capability_readers import generic_openai
from src.model_capability_readers.base import (
ModelCapabilityRecord,
VENDOR_OPENROUTER,
as_list,
as_mapping,
build_capability,
compact_str,
deterministic_controls_from_supported_parameters,
family_from_modalities,
int_limit,
merge_unique,
model_id_from,
modalities_from_value,
openai_model_items,
split_modality_arrow,
stable_model_id_for,
)
vendor = VENDOR_OPENROUTER
_SUPPORTED_PARAMETER_CAPS = {
"tools": mc.CAP_TOOL_CALL,
"tool_choice": mc.CAP_TOOL_CALL,
"function_calling": mc.CAP_TOOL_CALL,
"parallel_tool_calls": mc.CAP_TOOL_CALL,
"response_format": mc.CAP_JSON_MODE,
"structured_outputs": mc.CAP_STRUCTURED_OUTPUT,
"structured_output": mc.CAP_STRUCTURED_OUTPUT,
"reasoning": mc.CAP_REASONING,
"reasoning_effort": mc.CAP_REASONING,
"include_reasoning": mc.CAP_REASONING,
"web_search": mc.CAP_WEB_SEARCH,
"web_search_options": mc.CAP_WEB_SEARCH,
}
def _capabilities_from_supported_parameters(values: Any) -> tuple[str, ...]:
iterable = values if isinstance(values, list) else ()
out: list[str] = []
for value in iterable:
cap = _SUPPORTED_PARAMETER_CAPS.get(compact_str(value).lower().replace("-", "_"))
if cap and cap not in out:
out.append(cap)
return tuple(out)
def _limits_from_model(raw: Mapping[str, Any]) -> dict[str, Any]:
architecture = as_mapping(raw.get("architecture"))
top_provider = as_mapping(raw.get("top_provider"))
per_request_limits = as_mapping(raw.get("per_request_limits"))
limits: dict[str, Any] = {}
for key, canonical in (
("context_length", "context_tokens"),
("max_context_length", "context_tokens"),
("input_token_limit", "input_tokens"),
("output_token_limit", "output_tokens"),
("max_completion_tokens", "output_tokens"),
):
value = int_limit(raw.get(key) or architecture.get(key) or top_provider.get(key))
if value:
limits[canonical] = value
for key, value in per_request_limits.items():
limit = int_limit(value)
if limit:
limits[f"per_request_{key}"] = limit
return limits
def _has_supported_voices(value: Any) -> bool:
return any(compact_str(item) for item in as_list(value))
def _capabilities_from_modalities(
input_modalities: tuple[str, ...],
output_modalities: tuple[str, ...],
*,
supported_voices: Any = None,
) -> tuple[str, ...]:
input_set = set(input_modalities)
output_set = set(output_modalities)
capabilities: list[str] = []
if mc.MODALITY_IMAGE in input_set and mc.MODALITY_TEXT in output_set:
capabilities.append(mc.CAP_VISION)
if mc.MODALITY_FILE in input_set:
capabilities.append(mc.CAP_FILES)
if mc.MODALITY_PDF in input_set:
capabilities.append(mc.CAP_PDF)
if mc.MODALITY_AUDIO in input_set:
capabilities.append(mc.CAP_AUDIO_INPUT)
if mc.MODALITY_AUDIO in output_set:
capabilities.append(mc.CAP_AUDIO_OUTPUT)
if _has_supported_voices(supported_voices):
capabilities.append(mc.CAP_TTS)
if mc.MODALITY_IMAGE in output_set:
capabilities.append(mc.CAP_IMAGE_GENERATION)
if mc.MODALITY_IMAGE in input_set:
capabilities.append(mc.CAP_IMAGE_EDITING)
if mc.MODALITY_VIDEO in output_set:
capabilities.append(mc.CAP_VIDEO_GENERATION)
return tuple(capabilities)
def _default_parameter_controls(raw: Mapping[str, Any]) -> tuple[str, ...]:
defaults = as_mapping(raw.get("default_parameters"))
return tuple(key for key, value in defaults.items() if value is not None)
def _deterministic_controls_from_model(raw: Mapping[str, Any]) -> tuple[mc.DeterministicControl, ...]:
return deterministic_controls_from_supported_parameters(
merge_unique(
as_list(raw.get("supported_parameters")),
_default_parameter_controls(raw),
)
)
def record_from_model(
raw: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> ModelCapabilityRecord | None:
model_id = model_id_from(raw, "id", "name")
if not model_id:
return None
architecture = as_mapping(raw.get("architecture"))
input_modalities = modalities_from_value(
raw.get("input_modalities") or architecture.get("input_modalities")
)
output_modalities = modalities_from_value(
raw.get("output_modalities") or architecture.get("output_modalities")
)
if not input_modalities or not output_modalities:
arrow_input, arrow_output = split_modality_arrow(
raw.get("modality") or architecture.get("modality")
)
input_modalities = input_modalities or arrow_input
output_modalities = output_modalities or arrow_output
capabilities = list(_capabilities_from_supported_parameters(raw.get("supported_parameters")))
capabilities.extend(
_capabilities_from_modalities(
input_modalities,
output_modalities,
supported_voices=raw.get("supported_voices"),
)
)
family = family_from_modalities(input_modalities, output_modalities)
if family == mc.FAMILY_UNKNOWN:
fallback = generic_openai.record_from_model(
raw,
vendor_id=VENDOR_OPENROUTER,
endpoint_id=endpoint_id,
base_url=base_url,
)
return fallback
capability = build_capability(
family=family,
input_modalities=input_modalities,
output_modalities=output_modalities,
capabilities=merge_unique(capabilities),
limits=_limits_from_model(raw),
)
return ModelCapabilityRecord(
vendor=VENDOR_OPENROUTER,
model_id=model_id,
stable_model_id=stable_model_id_for(VENDOR_OPENROUTER, model_id, endpoint_id=endpoint_id, base_url=base_url),
display_name=compact_str(raw.get("name")) or model_id,
capability=capability,
deterministic_controls=_deterministic_controls_from_model(raw),
raw=raw,
)
def records_from_payload(
payload: Mapping[str, Any],
*,
endpoint_id: Any = "",
base_url: Any = "",
) -> tuple[ModelCapabilityRecord, ...]:
records: list[ModelCapabilityRecord] = []
for item in openai_model_items(payload):
record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)
if record:
records.append(record)
return tuple(records)
+9 -1
View File
@@ -934,6 +934,13 @@ function initEndpointForm() {
function _apiEndpointKind() { function _apiEndpointKind() {
return (kindSel && kindSel.value) ? kindSel.value : 'api'; return (kindSel && kindSel.value) ? kindSel.value : 'api';
} }
function _modelRefreshModeForApiEndpoint(url, endpointKind) {
if (endpointKind === 'proxy') return 'manual';
try {
if ((new URL(url)).hostname.toLowerCase() === 'generativelanguage.googleapis.com') return '';
} catch (_) {}
return 'auto';
}
function _normalizeBaseUrl(raw) { function _normalizeBaseUrl(raw) {
let u = raw.trim(); let u = raw.trim();
// Fix common protocol typos // Fix common protocol typos
@@ -1081,7 +1088,8 @@ function initEndpointForm() {
fd.append('base_url', url); fd.append('base_url', url);
const endpointKind = _apiEndpointKind(); const endpointKind = _apiEndpointKind();
fd.append('endpoint_kind', endpointKind); fd.append('endpoint_kind', endpointKind);
fd.append('model_refresh_mode', endpointKind === 'proxy' ? 'manual' : 'auto'); const refreshMode = _modelRefreshModeForApiEndpoint(url, endpointKind);
if (refreshMode) fd.append('model_refresh_mode', refreshMode);
fd.append('model_refresh_timeout', '30'); fd.append('model_refresh_timeout', '30');
if (apiKey) fd.append('api_key', apiKey); if (apiKey) fd.append('api_key', apiKey);
if (provider.value && provider.selectedOptions && provider.selectedOptions[0]) { if (provider.value && provider.selectedOptions && provider.selectedOptions[0]) {
+18
View File
@@ -28,6 +28,24 @@ def test_provider_selection_is_inert_and_add_button_starts_device_flow():
assert "_startProviderDeviceAuth(deviceAuthProvider" in add_block assert "_startProviderDeviceAuth(deviceAuthProvider" in add_block
def test_google_add_omits_auto_refresh_mode_for_backend_manual_default():
refresh_helper = _between(
_ADMIN,
"function _modelRefreshModeForApiEndpoint",
"function _normalizeBaseUrl",
)
add_block = _between(
_ADMIN,
"el('adm-epAddBtn').addEventListener('click'",
"async function _startProviderDeviceAuth",
)
assert "generativelanguage.googleapis.com" in refresh_helper
assert "return '';" in refresh_helper
assert "_modelRefreshModeForApiEndpoint(url, endpointKind)" in add_block
assert "if (refreshMode) fd.append('model_refresh_mode', refreshMode)" in add_block
def test_device_auth_selection_disables_and_dims_api_test_button(): def test_device_auth_selection_disables_and_dims_api_test_button():
form_block = _between(_ADMIN, "function _setApiFormForProvider()", "function _renderPickerMenu()") form_block = _between(_ADMIN, "function _setApiFormForProvider()", "function _renderPickerMenu()")
+248
View File
@@ -0,0 +1,248 @@
import src.model_capabilities as mc
def surfaces(capability):
return set(mc.display_surfaces_for(capability))
def test_endpoint_type_llm_maps_to_explicit_chat_capability():
capability = mc.capability_from_endpoint_type("llm")
assert capability.family == mc.FAMILY_CHAT
assert capability.primary_task == mc.TASK_CHAT_COMPLETIONS
assert capability.modalities.input == (mc.MODALITY_TEXT,)
assert capability.modalities.output == (mc.MODALITY_TEXT,)
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
assert capability.confidence == mc.CONFIDENCE_EXPLICIT
assert surfaces(capability) == {"chat"}
def test_endpoint_type_image_maps_to_explicit_image_generation_capability():
capability = mc.capability_from_endpoint_type("image")
assert capability.family == mc.FAMILY_IMAGE
assert capability.primary_task == mc.TASK_IMAGE_GENERATE
assert capability.modalities.output == (mc.MODALITY_IMAGE,)
assert capability.capabilities == (mc.CAP_IMAGE_GENERATION,)
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
assert capability.confidence == mc.CONFIDENCE_EXPLICIT
assert surfaces(capability) == {"image_generation"}
def test_missing_or_unknown_endpoint_type_does_not_imply_chat():
for model_type in (None, "", "openai-compatible", "text"):
capability = mc.capability_from_endpoint_type(model_type)
assert capability.family == mc.FAMILY_UNKNOWN
assert capability.primary_task == mc.TASK_UNKNOWN
assert capability.source == mc.SOURCE_ENDPOINT_CONFIG
assert mc.display_surfaces_for(capability) == ()
def test_provider_record_normalizes_aliases_and_boolean_capability_maps():
capability = mc.ModelCapability.from_dict(
{
"family": "llm",
"modalities": {
"input": ["text", "images", "docs", "images"],
"output": "text",
},
"capabilities": {
"tools": True,
"unknown_vendor_flag": True,
"vision": True,
"tts": False,
},
"limits": {"max_context_tokens": 32768, "": "ignored"},
"source": "provider_reader",
"confidence": "provider_reported",
}
)
assert capability.family == mc.FAMILY_CHAT
assert capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE)
assert capability.modalities.output == (mc.MODALITY_TEXT,)
assert capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION)
assert capability.limits == (("max_context_tokens", 32768),)
assert surfaces(capability) == {"chat", "vision_chat", "document_chat"}
assert capability.to_dict() == {
"family": mc.FAMILY_CHAT,
"primary_task": mc.TASK_CHAT_COMPLETIONS,
"modalities": {
"input": [mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE],
"output": [mc.MODALITY_TEXT],
},
"capabilities": [mc.CAP_TOOL_CALL, mc.CAP_VISION],
"limits": {"max_context_tokens": 32768},
"source": mc.SOURCE_PROVIDER_READER,
"confidence": mc.CONFIDENCE_PROVIDER_REPORTED,
}
def test_unknown_or_malformed_capability_record_stays_unknown():
assert mc.ModelCapability.from_dict(None).to_dict() == mc.unknown_capability().to_dict()
capability = mc.ModelCapability.build(
family="not-real",
primary_task=1234,
input_modalities=object(),
output_modalities=["text", "not-real"],
capabilities=["vision", "not-real"],
source="not-real",
confidence="not-real",
)
assert capability.family == mc.FAMILY_UNKNOWN
assert capability.primary_task == "1234"
assert capability.modalities.input == ()
assert capability.modalities.output == (mc.MODALITY_TEXT,)
assert capability.capabilities == (mc.CAP_VISION,)
assert capability.source == mc.SOURCE_UNKNOWN
assert capability.confidence == mc.CONFIDENCE_UNKNOWN
assert mc.display_surfaces_for(capability) == ()
def test_display_surface_queries_cover_core_model_categories():
assert surfaces(
mc.ModelCapability.build(
family=mc.FAMILY_IMAGE,
input_modalities=[mc.MODALITY_IMAGE],
output_modalities=[mc.MODALITY_IMAGE],
capabilities=[mc.CAP_INPAINTING],
)
) == {"image_editing"}
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_EMBEDDING)) == {"embeddings"}
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_RERANK)) == {"rerank_scoring"}
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_MODERATION)) == {"moderation_classification"}
assert surfaces(mc.ModelCapability.build(family=mc.FAMILY_CLASSIFICATION)) == {"moderation_classification"}
def test_audio_surface_matches_audio_input_or_output_when_capability_is_known():
transcription = mc.ModelCapability.build(
family=mc.FAMILY_AUDIO,
primary_task=mc.TASK_AUDIO_TRANSCRIBE,
input_modalities=[mc.MODALITY_AUDIO],
output_modalities=[mc.MODALITY_TEXT],
capabilities=[mc.CAP_TRANSCRIPTION],
)
synthesis = mc.ModelCapability.build(
family=mc.FAMILY_AUDIO,
primary_task=mc.TASK_AUDIO_SYNTHESIZE,
input_modalities=[mc.MODALITY_TEXT],
output_modalities=[mc.MODALITY_AUDIO],
capabilities=[mc.CAP_TTS],
)
assert surfaces(transcription) == {"audio_realtime"}
assert surfaces(synthesis) == {"audio_realtime"}
def test_capability_assertion_tracks_claimed_status_separately_from_capability_metadata():
assertion = mc.CapabilityAssertion.build(
capability="tools",
status="claimed",
source="provider_reader",
confidence="provider_reported",
evidence={"field": "supported_parameters"},
)
assert assertion.capability == mc.CAP_TOOL_CALL
assert assertion.status == mc.ASSERTION_CLAIMED
assert assertion.source == mc.SOURCE_PROVIDER_READER
assert assertion.confidence == mc.CONFIDENCE_PROVIDER_REPORTED
assert assertion.to_dict() == {
"capability": mc.CAP_TOOL_CALL,
"status": mc.ASSERTION_CLAIMED,
"source": mc.SOURCE_PROVIDER_READER,
"confidence": mc.CONFIDENCE_PROVIDER_REPORTED,
"evidence": {"field": "supported_parameters"},
"tested_at": "",
}
def test_capability_probe_result_converts_pass_and_fail_to_assertions():
passed = mc.CapabilityProbeResult.build(
provider="openrouter",
endpoint_id="ep-1",
model_id="vendor/model",
stable_model_id="openrouter|endpoint:ep-1|vendor/model",
capability="tool_calls",
status="pass",
tested_at="2026-06-04T20:00:00Z",
request_hash="abc123",
response_fingerprint="fp-test",
evidence={"contract": "single_fake_tool"},
)
failed = mc.CapabilityProbeResult.build(
provider="openrouter",
model_id="vendor/model",
capability="vision",
status="fail",
)
pass_assertion = passed.to_assertion()
fail_assertion = failed.to_assertion()
assert pass_assertion.capability == mc.CAP_TOOL_CALL
assert pass_assertion.status == mc.ASSERTION_VERIFIED
assert pass_assertion.source == mc.SOURCE_CAPABILITY_PROBE
assert pass_assertion.confidence == mc.CONFIDENCE_EXPLICIT
assert pass_assertion.tested_at == "2026-06-04T20:00:00Z"
assert dict(pass_assertion.evidence)["request_hash"] == "abc123"
assert dict(pass_assertion.evidence)["contract"] == "single_fake_tool"
assert fail_assertion.capability == mc.CAP_VISION
assert fail_assertion.status == mc.ASSERTION_UNSUPPORTED
assert fail_assertion.source == mc.SOURCE_CAPABILITY_PROBE
def test_deterministic_controls_are_normalized_as_claims_not_capabilities():
controls = mc.deterministic_controls_from_values(
["temp", "top-p", "top-k", "seed", "unknown"],
source=mc.SOURCE_PROVIDER_READER,
)
assert [control.control for control in controls] == [
mc.CONTROL_TEMPERATURE,
mc.CONTROL_TOP_P,
mc.CONTROL_TOP_K,
mc.CONTROL_SEED,
]
assert {control.status for control in controls} == {mc.ASSERTION_CLAIMED}
assert {control.source for control in controls} == {mc.SOURCE_PROVIDER_READER}
def test_reasoning_control_mechanisms_normalize_known_provider_shapes():
values = [
"think_directive",
"system_prompt_directive",
"enable_thinking",
"think_bool",
"reasoning_object",
"thinking_budget",
"reasoning_effort",
]
assert [mc.normalize_reasoning_control_mechanism(value) for value in values] == [
mc.REASONING_CONTROL_MESSAGE_DIRECTIVE,
mc.REASONING_CONTROL_SYSTEM_DIRECTIVE,
mc.REASONING_CONTROL_TEMPLATE_KWARG,
mc.REASONING_CONTROL_NATIVE_BOOL,
mc.REASONING_CONTROL_STRUCTURED_OBJECT,
mc.REASONING_CONTROL_BUDGET,
mc.REASONING_CONTROL_EFFORT,
]
def test_reasoning_control_values_can_describe_provider_supported_auto():
values = ["enabled", "disabled", "adaptive", "dynamic", "provider_auto"]
assert [mc.normalize_reasoning_control_value(value) for value in values] == [
mc.REASONING_CONTROL_VALUE_ON,
mc.REASONING_CONTROL_VALUE_OFF,
mc.REASONING_CONTROL_VALUE_AUTO,
mc.REASONING_CONTROL_VALUE_AUTO,
mc.REASONING_CONTROL_VALUE_AUTO,
]
assert mc.normalize_reasoning_control_value("message_directive") == ""
+646
View File
@@ -0,0 +1,646 @@
import src.model_capabilities as mc
import src.model_capability_readers as readers
from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter
from src.model_capability_readers.base import (
VENDOR_GENERIC_OPENAI,
VENDOR_GOOGLE,
VENDOR_LLAMACPP,
VENDOR_LMSTUDIO,
VENDOR_OLLAMA,
VENDOR_OPENAI,
VENDOR_OPENROUTER,
detect_vendor,
stable_model_id_for,
)
def surfaces(record):
return set(mc.display_surfaces_for(record.capability))
def test_detect_vendor_uses_endpoint_kind_then_host_and_common_local_ports():
assert detect_vendor("https://example.test/v1", endpoint_kind="ollama") == VENDOR_OLLAMA
assert detect_vendor("http://127.0.0.1:8080", endpoint_kind="llama_cpp") == VENDOR_LLAMACPP
assert detect_vendor("https://openrouter.ai/api/v1") == VENDOR_OPENROUTER
assert detect_vendor("https://api.openai.com/v1") == VENDOR_OPENAI
assert detect_vendor("https://generativelanguage.googleapis.com/v1beta/openai") == VENDOR_GOOGLE
assert detect_vendor("http://127.0.0.1:11434") == VENDOR_OLLAMA
assert detect_vendor("http://127.0.0.1:1234") == VENDOR_LMSTUDIO
assert detect_vendor("http://127.0.0.1:8080") == VENDOR_GENERIC_OPENAI
assert detect_vendor("http://localhost:7000/v1") == VENDOR_GENERIC_OPENAI
def test_generic_openai_reader_keeps_basic_model_payload_unknown():
records = generic_openai.records_from_payload(
{
"object": "list",
"data": [
{"id": "gpt-example", "object": "model", "owned_by": "vendor"},
],
}
)
assert len(records) == 1
record = records[0]
assert record.vendor == VENDOR_GENERIC_OPENAI
assert record.model_id == "gpt-example"
assert record.capability.family == mc.FAMILY_UNKNOWN
assert record.capability.source == mc.SOURCE_PROVIDER_READER
assert record.capability.confidence == mc.CONFIDENCE_UNKNOWN
assert record.stable_model_id == "generic_openai|global|gpt-example"
assert record.capability_assertions == ()
assert record.deterministic_controls == ()
assert surfaces(record) == set()
def test_stable_model_id_is_endpoint_scoped_for_local_or_configured_servers():
assert stable_model_id_for("ollama", "qwen:latest", endpoint_id="7") == "ollama|endpoint:7|qwen:latest"
assert stable_model_id_for("ollama", "qwen:latest", base_url="http://127.0.0.1:11434") != stable_model_id_for(
"ollama",
"qwen:latest",
base_url="http://10.0.0.12:11434",
)
def test_registry_uses_openai_reader_for_openai_vendor():
records = readers.records_from_payload({"data": [{"id": "shape-only-model"}]}, vendor=VENDOR_OPENAI)
assert len(records) == 1
assert records[0].vendor == VENDOR_OPENAI
assert records[0].stable_model_id == "openai|global|shape-only-model"
assert records[0].capability.family == mc.FAMILY_UNKNOWN
def test_openai_reader_keeps_official_model_shape_identity_only():
records = openai.records_from_payload(
{
"object": "list",
"data": [
{
"id": "shape-only-model",
"object": "model",
"created": 1700000000,
"owned_by": "openai",
}
],
}
)
assert len(records) == 1
record = records[0]
assert record.vendor == VENDOR_OPENAI
assert record.model_id == "shape-only-model"
assert record.capability.family == mc.FAMILY_UNKNOWN
assert record.capability.source == mc.SOURCE_PROVIDER_READER
assert record.capability.confidence == mc.CONFIDENCE_UNKNOWN
assert record.capability_assertions == ()
assert record.deterministic_controls == ()
assert surfaces(record) == set()
def test_registry_passes_endpoint_context_to_vendor_reader():
records = readers.records_from_payload(
{"data": [{"id": "local.gguf", "owned_by": "llamacpp"}]},
vendor=VENDOR_LLAMACPP,
base_url="http://localhost:8000",
)
assert len(records) == 1
assert records[0].stable_model_id == stable_model_id_for(
VENDOR_LLAMACPP,
"local.gguf",
base_url="http://localhost:8000",
)
def test_openrouter_reader_maps_rich_architecture_and_supported_parameters():
records = openrouter.records_from_payload(
{
"data": [
{
"id": "google/gemini-vision",
"name": "Gemini Vision",
"architecture": {"modality": "text+image->text"},
"supported_parameters": [
"tools",
"response_format",
"reasoning",
"include_reasoning",
"parallel_tool_calls",
"temperature",
"top_p",
"seed",
],
"context_length": 1048576,
"top_provider": {"max_completion_tokens": 65536},
},
{
"id": "black-forest-labs/flux",
"architecture": {"input_modalities": ["text"], "output_modalities": ["image"]},
},
{
"id": "vendor/image-edit-shape",
"architecture": {"input_modalities": ["text", "image", "file"], "output_modalities": ["text", "image"]},
"supported_parameters": ["structured_outputs", "web_search_options"],
},
{
"id": "vendor/audio-shape",
"architecture": {"input_modalities": ["text", "audio"], "output_modalities": ["text", "audio"]},
"supported_voices": ["alloy"],
"default_parameters": {"temperature": 0.7, "top_p": 0.9, "top_k": None},
"per_request_limits": {"prompt_tokens": 12000, "completion_tokens": 4000, "requests": "2"},
},
{
"id": "vendor/embedder",
"architecture": {"modality": "text->embedding"},
},
]
}
)
assert [record.model_id for record in records] == [
"google/gemini-vision",
"black-forest-labs/flux",
"vendor/image-edit-shape",
"vendor/audio-shape",
"vendor/embedder",
]
vision = records[0]
assert vision.capability.family == mc.FAMILY_CHAT
assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
assert vision.capability.capabilities == (
mc.CAP_TOOL_CALL,
mc.CAP_JSON_MODE,
mc.CAP_REASONING,
mc.CAP_VISION,
)
assert [(assertion.capability, assertion.status) for assertion in vision.capability_assertions] == [
(mc.CAP_TOOL_CALL, mc.ASSERTION_CLAIMED),
(mc.CAP_JSON_MODE, mc.ASSERTION_CLAIMED),
(mc.CAP_REASONING, mc.ASSERTION_CLAIMED),
(mc.CAP_VISION, mc.ASSERTION_CLAIMED),
]
assert [(control.control, control.status) for control in vision.deterministic_controls] == [
(mc.CONTROL_TEMPERATURE, mc.ASSERTION_CLAIMED),
(mc.CONTROL_TOP_P, mc.ASSERTION_CLAIMED),
(mc.CONTROL_SEED, mc.ASSERTION_CLAIMED),
]
assert dict(vision.capability.limits) == {"context_tokens": 1048576, "output_tokens": 65536}
assert surfaces(vision) == {"chat", "vision_chat"}
assert records[1].capability.family == mc.FAMILY_IMAGE
assert records[1].capability.capabilities == (mc.CAP_IMAGE_GENERATION,)
assert surfaces(records[1]) == {"image_generation"}
image_edit = records[2]
assert image_edit.capability.family == mc.FAMILY_IMAGE
assert image_edit.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE, mc.MODALITY_FILE)
assert image_edit.capability.modalities.output == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
assert image_edit.capability.capabilities == (
mc.CAP_STRUCTURED_OUTPUT,
mc.CAP_WEB_SEARCH,
mc.CAP_VISION,
mc.CAP_FILES,
mc.CAP_IMAGE_GENERATION,
mc.CAP_IMAGE_EDITING,
)
assert surfaces(image_edit) == {"image_generation", "image_editing"}
audio = records[3]
assert audio.capability.family == mc.FAMILY_AUDIO
assert audio.capability.capabilities == (mc.CAP_AUDIO_INPUT, mc.CAP_AUDIO_OUTPUT, mc.CAP_TTS)
assert dict(audio.capability.limits) == {
"per_request_completion_tokens": 4000,
"per_request_prompt_tokens": 12000,
"per_request_requests": 2,
}
assert [control.control for control in audio.deterministic_controls] == [
mc.CONTROL_TEMPERATURE,
mc.CONTROL_TOP_P,
]
assert surfaces(audio) == {"audio_realtime"}
assert records[4].capability.family == mc.FAMILY_EMBEDDING
assert surfaces(records[4]) == {"embeddings"}
def test_google_reader_maps_provider_fields_without_claiming_unreported_modalities():
records = google.records_from_payload(
{
"models": [
{
"name": "models/gemini-3.1-flash-image",
"displayName": "Gemini 3.1 Flash Image",
"supportedGenerationMethods": ["generateContent"],
"inputTokenLimit": 1000000,
"outputTokenLimit": 8192,
"thinking": True,
"temperature": 1.0,
"topP": 0.95,
"topK": 40,
},
{
"name": "models/text-embedding-example",
"supportedGenerationMethods": ["embedContent"],
},
]
}
)
assert len(records) == 2
content = records[0]
assert content.vendor == VENDOR_GOOGLE
assert content.model_id == "gemini-3.1-flash-image"
assert content.capability.family == mc.FAMILY_UNKNOWN
assert content.capability.modalities.input == ()
assert content.capability.modalities.output == ()
assert content.capability.capabilities == (mc.CAP_REASONING,)
assert dict(content.capability.limits) == {
"context_tokens": 1000000,
"input_tokens": 1000000,
"output_tokens": 8192,
}
assert [control.control for control in content.deterministic_controls] == [
mc.CONTROL_TEMPERATURE,
mc.CONTROL_TOP_P,
mc.CONTROL_TOP_K,
]
assert surfaces(content) == set()
embedding = records[1]
assert embedding.capability.family == mc.FAMILY_EMBEDDING
assert surfaces(embedding) == {"embeddings"}
def test_google_ai_studio_mapping_does_not_infer_media_from_model_names():
records = google.records_from_payload(
{
"models": [
{
"name": "models/imagen-4.0-generate-001",
"displayName": "Imagen 4",
"supportedGenerationMethods": ["predict"],
},
{
"name": "models/veo-3.1-generate-preview",
"displayName": "Veo 3.1",
"supportedGenerationMethods": ["predictLongRunning"],
},
{
"name": "models/gemini-3.1-flash-tts-preview",
"supportedGenerationMethods": ["generateContent", "countTokens", "createCachedContent", "batchGenerateContent"],
},
{
"name": "models/lyria-3-pro-preview",
"displayName": "Lyria 3 Pro Preview",
"supportedGenerationMethods": ["generateContent", "countTokens"],
},
]
}
)
assert len(records) == 4
assert [record.capability.family for record in records] == [
mc.FAMILY_UNKNOWN,
mc.FAMILY_UNKNOWN,
mc.FAMILY_UNKNOWN,
mc.FAMILY_UNKNOWN,
]
assert all(record.capability.modalities.input == () for record in records)
assert all(record.capability.modalities.output == () for record in records)
assert all(surfaces(record) == set() for record in records)
assert [control.control for control in records[2].deterministic_controls] == [
mc.CONTROL_PROMPT_CACHING,
mc.CONTROL_BATCH,
]
def test_google_ai_studio_mapping_keeps_unrecognized_predict_models_unknown():
records = google.records_from_payload(
{
"models": [
{
"name": "models/vendor-future-media-001",
"supportedGenerationMethods": ["predict"],
}
]
}
)
assert len(records) == 1
assert records[0].capability.family == mc.FAMILY_UNKNOWN
assert surfaces(records[0]) == set()
def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown():
vision = ollama.record_from_show_payload(
"llava:latest",
{
"capabilities": ["completion", "vision", "tools"],
"model_info": {"llama.context_length": 4096},
},
)
embedding = ollama.record_from_show_payload(
"nomic-embed-text:latest",
{"capabilities": ["embedding"]},
)
tags = ollama.records_from_tags_payload({"models": [{"name": "qwen3:latest"}]})
assert vision is not None
assert vision.capability.family == mc.FAMILY_CHAT
assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
assert vision.capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL)
assert dict(vision.capability.limits) == {"context_tokens": 4096}
assert surfaces(vision) == {"chat", "vision_chat"}
assert embedding is not None
assert embedding.capability.family == mc.FAMILY_EMBEDDING
assert surfaces(embedding) == {"embeddings"}
assert len(tags) == 1
assert tags[0].capability.family == mc.FAMILY_UNKNOWN
assert surfaces(tags[0]) == set()
def test_ollama_reader_uses_show_shape_without_architecture_name_matching():
record = ollama.record_from_show_payload(
"local:latest",
{
"capabilities": ["completion", "thinking", "tools"],
"parameters": "temperature 0.7\nnum_ctx 8192",
"model_info": {
"future_architecture.context_length": 32768,
"future_architecture.embedding_length": 4096,
},
},
)
assert record is not None
assert record.capability.family == mc.FAMILY_CHAT
assert record.capability.modalities.input == (mc.MODALITY_TEXT,)
assert record.capability.modalities.output == (mc.MODALITY_TEXT,)
assert record.capability.capabilities == (mc.CAP_REASONING, mc.CAP_TOOL_CALL)
assert dict(record.capability.limits) == {"context_tokens": 8192}
assert surfaces(record) == {"chat"}
def test_ollama_reader_uses_generic_model_info_context_length_when_no_num_ctx():
record = ollama.record_from_show_payload(
"local:latest",
{
"capabilities": ["completion"],
"model_info": {"future_architecture.context_length": 32768},
},
)
assert record is not None
assert record.capability.family == mc.FAMILY_CHAT
assert dict(record.capability.limits) == {"context_tokens": 32768}
def test_lmstudio_reader_uses_native_v1_capabilities_when_present():
records = lmstudio.records_from_payload(
{
"models": [
{
"type": "llm",
"key": "google/gemma-vl",
"display_name": "Gemma VL",
"capabilities": {
"vision": True,
"trained_for_tool_use": True,
"reasoning": {"allowed_options": ["off", "on"], "default": "on"},
},
"loaded_instances": [
{"config": {"context_length": 8192}},
{"config": {"context_length": 4096}},
],
"max_context_length": 262144,
},
{
"type": "embedding",
"key": "nomic/embed",
},
{"key": "shape-without-type"},
]
}
)
assert len(records) == 3
vision = records[0]
assert vision.vendor == VENDOR_LMSTUDIO
assert vision.model_id == "google/gemma-vl"
assert vision.display_name == "Gemma VL"
assert vision.capability.family == mc.FAMILY_CHAT
assert vision.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE)
assert vision.capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL, mc.CAP_REASONING)
assert dict(vision.capability.limits) == {"context_tokens": 4096, "max_context_tokens": 262144}
assert surfaces(vision) == {"chat", "vision_chat"}
assert records[1].capability.family == mc.FAMILY_EMBEDDING
assert surfaces(records[1]) == {"embeddings"}
assert records[2].capability.family == mc.FAMILY_UNKNOWN
assert surfaces(records[2]) == set()
def test_lmstudio_reader_uses_legacy_native_v0_shape_for_family_and_limits():
records = lmstudio.records_from_payload(
{
"data": [
{
"id": "local-gemma",
"type": "llm",
"arch": "gemma3",
"loaded_context_length": 16384,
"max_context_length": 32768,
},
{
"id": "text-embedding-local",
"type": "embeddings",
"max_context_length": 2048,
},
]
}
)
assert len(records) == 2
chat = records[0]
assert chat.vendor == VENDOR_LMSTUDIO
assert chat.capability.family == mc.FAMILY_CHAT
assert chat.capability.modalities.input == (mc.MODALITY_TEXT,)
assert chat.capability.capabilities == ()
assert dict(chat.capability.limits) == {"context_tokens": 16384, "max_context_tokens": 32768}
assert surfaces(chat) == {"chat"}
assert records[1].capability.family == mc.FAMILY_EMBEDDING
assert dict(records[1].capability.limits) == {"context_tokens": 2048}
assert surfaces(records[1]) == {"embeddings"}
def test_lmstudio_openai_compatible_model_list_remains_identity_only():
records = lmstudio.records_from_payload(
{
"object": "list",
"data": [
{"id": "local-gemma-3-270m-it-qat-q4_k_m", "object": "model", "owned_by": "organization_owner"},
{"id": "text-embedding-nomic-embed-text-v1.5", "object": "model", "owned_by": "organization_owner"},
],
}
)
assert len(records) == 2
for record in records:
assert record.vendor == VENDOR_LMSTUDIO
assert record.capability.family == mc.FAMILY_UNKNOWN
assert record.capability_assertions == ()
assert surfaces(record) == set()
def test_lmstudio_unexpected_native_endpoint_error_yields_no_records():
assert lmstudio.records_from_payload({"error": "Unexpected endpoint or method. (GET /api/v1/models)"}) == ()
def test_llamacpp_reader_merges_models_props_and_slots_payloads():
models_payload = {
"object": "list",
"data": [
{
"id": "gemma-4-E2B-it-Q8_0.gguf",
"owned_by": "llamacpp",
"meta": {
"n_ctx_train": 131072,
"n_params": 4647450147,
"size": 5032532108,
},
}
],
"models": [
{
"name": "gemma-4-E2B-it-Q8_0.gguf",
"model": "gemma-4-E2B-it-Q8_0.gguf",
"capabilities": ["completion"],
"details": {"format": "gguf"},
}
],
}
props_payload = {
"model_alias": "gemma-4-E2B-it-Q8_0.gguf",
"model_path": "/models/gemma-4-E2B-it-Q8_0.gguf",
"build_info": "b1-c8ac02f",
"total_slots": 4,
"modalities": {"vision": False, "audio": False},
"chat_template_caps": {
"supports_object_arguments": True,
"supports_parallel_tool_calls": True,
"supports_preserve_reasoning": False,
"supports_string_content": True,
"supports_system_role": True,
"supports_tool_calls": True,
"supports_tools": True,
"supports_typed_content": False,
},
"default_generation_settings": {
"n_ctx": 16384,
"params": {
"temperature": 1.0,
"top_p": 0.95,
"seed": 4294967295,
"stream": True,
"samplers": ["top_p", "temperature"],
},
},
}
slots_payload = [
{"id": 0, "n_ctx": 16384, "speculative": False, "is_processing": False},
{"id": 1, "n_ctx": 16384, "speculative": False, "is_processing": False},
{"id": 2, "n_ctx": 16384, "speculative": False, "is_processing": False},
{"id": 3, "n_ctx": 16384, "speculative": False, "is_processing": False},
]
records = llamacpp.records_from_payloads(
models_payload=models_payload,
props_payload=props_payload,
slots_payload=slots_payload,
base_url="http://localhost:8000",
)
assert len(records) == 1
record = records[0]
assert record.vendor == VENDOR_LLAMACPP
assert record.model_id == "gemma-4-E2B-it-Q8_0.gguf"
assert record.stable_model_id == stable_model_id_for(
VENDOR_LLAMACPP,
"gemma-4-E2B-it-Q8_0.gguf",
base_url="http://localhost:8000",
)
assert record.capability.family == mc.FAMILY_CHAT
assert record.capability.modalities.input == (mc.MODALITY_TEXT,)
assert record.capability.modalities.output == (mc.MODALITY_TEXT,)
assert record.capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_STREAMING)
assert dict(record.capability.limits) == {
"context_tokens": 16384,
"model_bytes": 5032532108,
"parallel_slots": 4,
"parameters": 4647450147,
"training_context_tokens": 131072,
}
assert surfaces(record) == {"chat"}
assertion_status = {assertion.capability: assertion.status for assertion in record.capability_assertions}
assert assertion_status[mc.CAP_TOOL_CALL] == mc.ASSERTION_CLAIMED
assert assertion_status[mc.CAP_STREAMING] == mc.ASSERTION_CLAIMED
assert assertion_status[mc.CAP_VISION] == mc.ASSERTION_UNSUPPORTED
assert assertion_status[mc.CAP_AUDIO_INPUT] == mc.ASSERTION_UNSUPPORTED
assert mc.CAP_REASONING not in assertion_status
controls = {control.control: control.status for control in record.deterministic_controls}
assert controls == {
mc.CONTROL_TEMPERATURE: mc.ASSERTION_CLAIMED,
mc.CONTROL_TOP_P: mc.ASSERTION_CLAIMED,
mc.CONTROL_SEED: mc.ASSERTION_CLAIMED,
mc.CONTROL_SYSTEM_PROMPT: mc.ASSERTION_CLAIMED,
mc.CONTROL_TOOL_CHOICE: mc.ASSERTION_CLAIMED,
}
def test_llamacpp_openai_model_list_without_native_capability_shape_stays_unknown():
records = llamacpp.records_from_payload(
{
"object": "list",
"data": [
{
"id": "local-chat.gguf",
"owned_by": "llamacpp",
}
],
}
)
assert len(records) == 1
assert records[0].capability.family == mc.FAMILY_UNKNOWN
assert records[0].capability.capabilities == ()
assert records[0].capability_assertions == ()
assert surfaces(records[0]) == set()
def test_llamacpp_props_payload_reports_unsupported_modalities_without_model_list():
records = llamacpp.records_from_payload(
{
"model_alias": "local.gguf",
"modalities": {"vision": False, "audio": False},
"chat_template_caps": {"supports_tools": False, "supports_preserve_reasoning": False},
"default_generation_settings": {"n_ctx": 4096, "params": {"stream": True}},
}
)
assert len(records) == 1
record = records[0]
assert record.capability.family == mc.FAMILY_CHAT
assert record.capability.capabilities == (mc.CAP_STREAMING,)
assert {a.capability: a.status for a in record.capability_assertions} == {
mc.CAP_STREAMING: mc.ASSERTION_CLAIMED,
mc.CAP_VISION: mc.ASSERTION_UNSUPPORTED,
mc.CAP_AUDIO_INPUT: mc.ASSERTION_UNSUPPORTED,
}
+122
View File
@@ -48,6 +48,9 @@ with preserve_import_state("core.database", "src.database", "core.session_manage
_ping_endpoint, _ping_endpoint,
_parse_model_list, _parse_model_list,
_normalize_refresh_mode, _normalize_refresh_mode,
_normalize_endpoint_refresh_mode,
_endpoint_refresh_mode,
_is_google_api_base,
_truthy, _truthy,
_speech_settings_using_endpoint, _speech_settings_using_endpoint,
_clear_speech_settings_for_endpoint, _clear_speech_settings_for_endpoint,
@@ -464,6 +467,28 @@ class TestClassifyEndpoint:
assert _normalize_refresh_mode("manual", "proxy") == "manual" assert _normalize_refresh_mode("manual", "proxy") == "manual"
assert _normalize_refresh_mode("auto", "api") == "auto" assert _normalize_refresh_mode("auto", "api") == "auto"
def test_google_refresh_mode_defaults_manual_unless_explicit(self):
base = "https://generativelanguage.googleapis.com/v1beta/openai"
assert _normalize_endpoint_refresh_mode("", "api", base) == "manual"
assert _normalize_endpoint_refresh_mode(None, "auto", base) == "manual"
assert _normalize_endpoint_refresh_mode("auto", "api", base) == "auto"
def test_only_gemini_native_host_uses_google_models_api(self):
assert _is_google_api_base("https://generativelanguage.googleapis.com/v1beta/openai") is True
assert _is_google_api_base(
"https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/endpoints/openapi"
) is False
def test_existing_google_endpoint_refresh_mode_defaults_manual(self):
ep = SimpleNamespace(
model_refresh_mode=None,
endpoint_kind="api",
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
)
assert _endpoint_refresh_mode(ep, "api") == "manual"
ep.model_refresh_mode = "auto"
assert _endpoint_refresh_mode(ep, "api") == "auto"
def test_parse_model_list_accepts_json_and_text(self): def test_parse_model_list_accepts_json_and_text(self):
assert _parse_model_list('["a", "b", "a"]') == ["a", "b"] assert _parse_model_list('["a", "b", "a"]') == ["a", "b"]
assert _parse_model_list("a, b\nc") == ["a", "b", "c"] assert _parse_model_list("a, b\nc") == ["a", "b", "c"]
@@ -568,6 +593,83 @@ class TestSetupProbeSafety:
assert _probe_endpoint("https://api.groq.com/openai/v1") == _PROVIDER_CURATED["groq"] assert _probe_endpoint("https://api.groq.com/openai/v1") == _PROVIDER_CURATED["groq"]
def test_google_probe_uses_native_paginated_models_api(self, monkeypatch):
monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False)
monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/"))
seen = []
def fake_get(url, headers=None, params=None, timeout=None, verify=None, **kwargs):
seen.append((url, headers, params, timeout, verify))
request = httpx.Request("GET", url)
page_token = (params or {}).get("pageToken")
if page_token:
return httpx.Response(
200,
request=request,
json={
"models": [{
"name": "models/gemini-page-two",
"supportedGenerationMethods": ["generateContent"],
}]
},
)
return httpx.Response(
200,
request=request,
json={
"models": [
{
"name": "models/gemini-page-one",
"supportedGenerationMethods": ["generateContent"],
},
{
"baseModelId": "gemini-base-id",
"name": "models/ignored-version",
"supportedGenerationMethods": ["generateText"],
},
{
"name": "models/imagen-4.0-generate-001",
"supportedGenerationMethods": ["predict"],
},
{
"name": "models/text-embedding-example",
"supportedGenerationMethods": ["embedContent"],
},
{"name": "models/missing-method-metadata"},
],
"nextPageToken": "next-page",
},
)
monkeypatch.setattr(model_routes.httpx, "get", fake_get)
assert _probe_endpoint("https://generativelanguage.googleapis.com/v1beta/openai", "google-key") == [
"gemini-page-one",
"gemini-base-id",
"gemini-page-two",
]
assert [call[0] for call in seen] == [
"https://generativelanguage.googleapis.com/v1beta/models",
"https://generativelanguage.googleapis.com/v1beta/models",
]
assert seen[0][1] == {"Accept": "application/json", "x-goog-api-key": "google-key"}
assert seen[0][2] == {"pageSize": 1000}
assert seen[1][1] == {"Accept": "application/json", "x-goog-api-key": "google-key"}
assert seen[1][2] == {"pageSize": 1000, "pageToken": "next-page"}
def test_google_probe_does_not_use_curated_fallback_on_failure(self, monkeypatch):
monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False)
monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/"))
def fake_get(url, headers=None, params=None, timeout=None, verify=None, **kwargs):
request = httpx.Request("GET", url)
response = httpx.Response(401, request=request)
raise httpx.HTTPStatusError("unauthorized", request=request, response=response)
monkeypatch.setattr(model_routes.httpx, "get", fake_get)
assert _probe_endpoint("https://generativelanguage.googleapis.com/v1beta/openai", "bad-key") == []
def test_keyed_anthropic_probe_does_not_fallback_on_failure(self, monkeypatch): def test_keyed_anthropic_probe_does_not_fallback_on_failure(self, monkeypatch):
monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False) monkeypatch.setattr(endpoint_resolver, "resolve_url", lambda url: url, raising=False)
monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/")) monkeypatch.setattr(model_routes, "_normalize_base", lambda url: url.rstrip("/"))
@@ -1101,6 +1203,26 @@ def test_post_creates_endpoint_with_pinned_models(monkeypatch):
assert json.loads(db.added[0].pinned_models) == ["deploy-1", "deploy-2"] assert json.loads(db.added[0].pinned_models) == ["deploy-1", "deploy-2"]
def test_post_google_endpoint_defaults_to_manual_refresh_when_mode_omitted(monkeypatch):
db = _PinnedFakeDb([])
_patch_create_deps(monkeypatch, db)
monkeypatch.setattr(model_routes, "_probe_endpoint", lambda *args, **kwargs: ["gemini-test"])
create = _get_route("/api/model-endpoints", "POST")
create(
_PinnedFakeRequest(),
base_url="https://generativelanguage.googleapis.com/v1beta/openai",
**_create_form_kwargs(
api_key="google-key",
endpoint_kind="api",
model_refresh_mode="",
),
)
assert len(db.added) == 1
assert db.added[0].model_refresh_mode == "manual"
def test_post_dedupe_existing_merges_and_returns_pinned(monkeypatch): def test_post_dedupe_existing_merges_and_returns_pinned(monkeypatch):
existing = _make_endpoint( existing = _make_endpoint(
base_url="http://host:1234/v1", base_url="http://host:1234/v1",